@hawkeyexl/manni 4.2.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +2 -2
  2. package/dist/batch-7JJWXWSF.js +18 -0
  3. package/dist/batch-7JJWXWSF.js.map +1 -0
  4. package/dist/calibrate-IA2ARFMH.js +13 -0
  5. package/dist/calibrate-IA2ARFMH.js.map +1 -0
  6. package/dist/capture-C32JXIFN.js +15 -0
  7. package/dist/capture-C32JXIFN.js.map +1 -0
  8. package/dist/chunk-3R23C5DT.js +120 -0
  9. package/dist/chunk-3R23C5DT.js.map +1 -0
  10. package/dist/chunk-CI2GOU3A.js +4447 -0
  11. package/dist/chunk-CI2GOU3A.js.map +1 -0
  12. package/dist/{chunk-TH565O2N.js → chunk-DP2ZA3WH.js} +898 -278
  13. package/dist/chunk-DP2ZA3WH.js.map +1 -0
  14. package/dist/chunk-ESJ5W7EV.js +297 -0
  15. package/dist/chunk-ESJ5W7EV.js.map +1 -0
  16. package/dist/chunk-IOYXPBCL.js +37 -0
  17. package/dist/chunk-IOYXPBCL.js.map +1 -0
  18. package/dist/chunk-KTSJ2ZHA.js +40 -0
  19. package/dist/chunk-KTSJ2ZHA.js.map +1 -0
  20. package/dist/{chunk-BNHQWIAE.js → chunk-MFDD3YGU.js} +2 -2
  21. package/dist/chunk-MOQXWXSG.js +1941 -0
  22. package/dist/chunk-MOQXWXSG.js.map +1 -0
  23. package/dist/{chunk-UYDRDFYF.js → chunk-O6RVQKK6.js} +1943 -909
  24. package/dist/chunk-O6RVQKK6.js.map +1 -0
  25. package/dist/chunk-QROCFW5S.js +174 -0
  26. package/dist/chunk-QROCFW5S.js.map +1 -0
  27. package/dist/chunk-RVVCA4WL.js +379 -0
  28. package/dist/chunk-RVVCA4WL.js.map +1 -0
  29. package/dist/cli.js +422 -42
  30. package/dist/cli.js.map +1 -1
  31. package/dist/docmeta.js +3 -2
  32. package/dist/docmeta.js.map +1 -1
  33. package/dist/index.d.ts +3873 -795
  34. package/dist/index.js +292 -4
  35. package/dist/index.js.map +1 -1
  36. package/dist/picker-6HRMETE7.js +12 -0
  37. package/dist/picker-6HRMETE7.js.map +1 -0
  38. package/package.json +3 -1
  39. package/plugin/tracevals/.claude-plugin/plugin.json +9 -0
  40. package/plugin/tracevals/hooks/hooks.json +17 -0
  41. package/dist/chunk-TH565O2N.js.map +0 -1
  42. package/dist/chunk-UYDRDFYF.js.map +0 -1
  43. /package/dist/{chunk-BNHQWIAE.js.map → chunk-MFDD3YGU.js.map} +0 -0
package/dist/cli.js CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  buildProgram
4
- } from "./chunk-BNHQWIAE.js";
4
+ } from "./chunk-MFDD3YGU.js";
5
5
  import {
6
6
  ALL_FILL_FIELDS,
7
7
  CONFIG_SECTION,
@@ -9,7 +9,6 @@ import {
9
9
  DEFAULT_CONFIG_FILENAME2 as DEFAULT_CONFIG_FILENAME,
10
10
  DEFAULT_EXTENSIONS,
11
11
  DocevalsError,
12
- EXECUTION_GRANTS,
13
12
  EXPORT_TARGETS,
14
13
  GraphError,
15
14
  INFER_FORMATS,
@@ -52,6 +51,7 @@ import {
52
51
  renderFill,
53
52
  renderFill2,
54
53
  renderList,
54
+ renderList2,
55
55
  renderQuery,
56
56
  renderRemoveJson,
57
57
  renderRemovePretty,
@@ -72,12 +72,14 @@ import {
72
72
  runExport,
73
73
  runFill,
74
74
  runFill2,
75
+ runFill3,
75
76
  runGenerate,
76
77
  runInit,
77
78
  runKeyRotate,
78
79
  runKeySet,
79
80
  runLint,
80
81
  runList,
82
+ runList2,
81
83
  runPromote,
82
84
  runQuery,
83
85
  runRemove,
@@ -97,7 +99,19 @@ import {
97
99
  shortSrc,
98
100
  spellSource,
99
101
  validateGraph
100
- } from "./chunk-UYDRDFYF.js";
102
+ } from "./chunk-O6RVQKK6.js";
103
+ import {
104
+ CALIBRATE_FORMATS,
105
+ REPORT_FORMATS as REPORT_FORMATS3,
106
+ SUMMARY_FORMATS as SUMMARY_FORMATS2,
107
+ parseFormat as parseFormat2,
108
+ runRun as runRun2
109
+ } from "./chunk-CI2GOU3A.js";
110
+ import "./chunk-QROCFW5S.js";
111
+ import {
112
+ allowExecutionMessage,
113
+ isExecutionGrant
114
+ } from "./chunk-MOQXWXSG.js";
101
115
  import {
102
116
  COMMON_FORMAT_LIST,
103
117
  CONCEPT_FIELDS,
@@ -109,11 +123,10 @@ import {
109
123
  STDIN_LABEL,
110
124
  STDIN_TOKEN,
111
125
  TERM_RELATION_FIELDS,
112
- ToolError,
113
126
  assertNonEmpty,
114
127
  collect,
128
+ colorFor,
115
129
  configOption,
116
- errorMessage,
117
130
  escapeWorkflowCommandMessage,
118
131
  escapeWorkflowCommandProperty,
119
132
  explicitFalse,
@@ -151,10 +164,17 @@ import {
151
164
  valueAt,
152
165
  warn,
153
166
  writeTextAtomic
154
- } from "./chunk-TH565O2N.js";
167
+ } from "./chunk-DP2ZA3WH.js";
168
+ import {
169
+ TracevalsError
170
+ } from "./chunk-3R23C5DT.js";
171
+ import {
172
+ ToolError,
173
+ errorMessage
174
+ } from "./chunk-IOYXPBCL.js";
155
175
 
156
176
  // src/cli.ts
157
- import { Command as Command8 } from "commander";
177
+ import { Command as Command9 } from "commander";
158
178
 
159
179
  // src/a11y/cli.ts
160
180
  import { Command } from "commander";
@@ -1375,7 +1395,7 @@ function renderCheckJunit(run) {
1375
1395
  // src/cite/cli.ts
1376
1396
  var UPDATE_FORMATS = ["pretty", "json"];
1377
1397
  var REMOVE_FORMATS = UPDATE_FORMATS;
1378
- function colorFor(command, isTTY, env) {
1398
+ function colorFor2(command, isTTY, env) {
1379
1399
  const noColor = colorOwner(command).opts().color === false;
1380
1400
  return shouldColor({ noColor, isTTY, env });
1381
1401
  }
@@ -1502,7 +1522,7 @@ function buildProgram3() {
1502
1522
  switch (format) {
1503
1523
  case "pretty":
1504
1524
  text2 = renderCheckPretty(run, {
1505
- color: colorFor(command, process.stdout.isTTY),
1525
+ color: colorFor2(command, process.stdout.isTTY),
1506
1526
  quiet: Boolean(options.quiet),
1507
1527
  showDiff: Boolean(options.showDiff),
1508
1528
  reveal: Boolean(options.reveal)
@@ -1666,7 +1686,7 @@ function buildProgram3() {
1666
1686
  dryRun
1667
1687
  });
1668
1688
  const text2 = format === "json" ? renderUpdateJson(run) : renderUpdatePretty(run, {
1669
- color: colorFor(command, process.stdout.isTTY),
1689
+ color: colorFor2(command, process.stdout.isTTY),
1670
1690
  // The diffs are the point of a dry run; a real run prints
1671
1691
  // what it rewrote and leaves the file to say the rest.
1672
1692
  showDiff: dryRun
@@ -1741,7 +1761,7 @@ function buildProgram3() {
1741
1761
  dryRun
1742
1762
  });
1743
1763
  const text2 = format === "json" ? renderRemoveJson(run) : renderRemovePretty(run, {
1744
- color: colorFor(command, process.stdout.isTTY),
1764
+ color: colorFor2(command, process.stdout.isTTY),
1745
1765
  // The diffs are the point of a dry run; a real run prints
1746
1766
  // what it removed and leaves the file to say the rest.
1747
1767
  showDiff: dryRun,
@@ -1766,28 +1786,14 @@ function buildProgram3() {
1766
1786
 
1767
1787
  // src/docevals/cli.ts
1768
1788
  import { Command as Command3 } from "commander";
1769
- function colorFor2(command, isTTY, env) {
1770
- const noColor = colorOwner2(command).opts().color === false;
1771
- return shouldColor({ noColor, isTTY, env });
1772
- }
1773
- function colorOwner2(command) {
1774
- for (let c = command; c !== null; c = c.parent) {
1775
- if (c.options.some((o) => o.long === "--no-color")) return c;
1776
- }
1777
- return command;
1778
- }
1779
1789
  function buildProgram4() {
1780
1790
  const program = new Command3();
1781
1791
  program.name("docevals").description(
1782
1792
  "Deterministic and LLM-as-judge evals for documentation pages, driven by frontmatter."
1783
1793
  ).version(package_default.version).option("--no-color", "disable colored output").showHelpAfterError("(add --help for usage)").exitOverride();
1784
1794
  function collectGrant(value, previous) {
1785
- if (!EXECUTION_GRANTS.includes(value)) {
1786
- fail(
1787
- new DocevalsError(
1788
- `--allow-execution must be one of ${EXECUTION_GRANTS.join(" | ")}, got "${value}"`
1789
- )
1790
- );
1795
+ if (!isExecutionGrant(value)) {
1796
+ fail(new DocevalsError(allowExecutionMessage(value)));
1791
1797
  }
1792
1798
  return [...previous, value];
1793
1799
  }
@@ -1860,7 +1866,7 @@ function buildProgram4() {
1860
1866
  });
1861
1867
  console.log(
1862
1868
  renderList(run, opts.format, {
1863
- color: colorFor2(command, process.stdout.isTTY)
1869
+ color: colorFor(command, process.stdout.isTTY)
1864
1870
  })
1865
1871
  );
1866
1872
  process.exitCode = run.exitCode;
@@ -1883,7 +1889,7 @@ function buildProgram4() {
1883
1889
  // frontmatter-commands docs/**` would silently swallow the glob and run
1884
1890
  // over the default file set instead. Repeat the flag to grant twice.
1885
1891
  "--allow-execution <kind>",
1886
- "Grant content-authored execution: frontmatter-commands",
1892
+ "Run only these execution grants (repeatable): frontmatter-commands",
1887
1893
  collectGrant,
1888
1894
  []
1889
1895
  ).option("--no-execution", "Clear every execution grant for this run").option("--no-generate", "Do not generate scripts for command evals missing a command").option("--no-cache", "Bypass the judge response cache").option("--fail-on-review", "Exit 1 when any eval lands in the human-review zone").option("--provider <name>", "Judge provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).option("--runs <n>", "Ensemble runs per eval", parseIntArg("--runs")).option(
@@ -1898,6 +1904,9 @@ function buildProgram4() {
1898
1904
  ).option("--suite <name>", "Run only evals in this suite").option(
1899
1905
  "--since <ref>",
1900
1906
  "Evaluate only pages whose file or eval manifest changed between this git ref and HEAD"
1907
+ ).option(
1908
+ "--newer-than <duration>",
1909
+ "Evaluate only pages whose file or eval manifest changed within this window, such as 30m, 24h, 7d or 2w; in CI, prefer --since"
1901
1910
  ).option(
1902
1911
  "--max-turns <n>",
1903
1912
  "Stop after this many ensemble runs (a cached ensemble costs none)",
@@ -1921,7 +1930,9 @@ function buildProgram4() {
1921
1930
  format: opts.format,
1922
1931
  deterministicOnly: opts.deterministicOnly,
1923
1932
  aiOnly: opts.aiOnly,
1924
- allowExecution: opts.allowExecution,
1933
+ // Commander's default is `[]`, which is "no flag", not "run nothing":
1934
+ // only a list someone typed narrows the run.
1935
+ allowExecution: opts.allowExecution.length > 0 ? opts.allowExecution : void 0,
1925
1936
  execution: opts.execution,
1926
1937
  generate: opts.generate,
1927
1938
  cache: opts.cache,
@@ -1935,6 +1946,7 @@ function buildProgram4() {
1935
1946
  evalNames: opts.eval,
1936
1947
  suite: opts.suite,
1937
1948
  since: opts.since,
1949
+ newerThan: opts.newerThan,
1938
1950
  // commander collapses `--baseline` to true and `--no-baseline` to
1939
1951
  // false on the same key; a string is an explicit path.
1940
1952
  baseline: opts.baseline,
@@ -1943,7 +1955,7 @@ function buildProgram4() {
1943
1955
  });
1944
1956
  console.log(
1945
1957
  render(report, opts.format, {
1946
- color: colorFor2(command, process.stdout.isTTY)
1958
+ color: colorFor(command, process.stdout.isTTY)
1947
1959
  })
1948
1960
  );
1949
1961
  process.exitCode = report.exitCode;
@@ -1959,7 +1971,7 @@ function buildProgram4() {
1959
1971
  ).option("--provider <name>", "Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).action(
1960
1972
  async (paths, opts, command) => {
1961
1973
  try {
1962
- const pc = palette(colorFor2(command, process.stdout.isTTY));
1974
+ const pc = palette(colorFor(command, process.stdout.isTTY));
1963
1975
  const result = await runGenerate(paths, {
1964
1976
  ...documentOptions(opts),
1965
1977
  provider: opts.provider,
@@ -2038,7 +2050,7 @@ function buildProgram4() {
2038
2050
  });
2039
2051
  console.log(
2040
2052
  renderFill(report, opts.format, {
2041
- color: colorFor2(command, process.stdout.isTTY)
2053
+ color: colorFor(command, process.stdout.isTTY)
2042
2054
  })
2043
2055
  );
2044
2056
  process.exitCode = report.exitCode;
@@ -2054,7 +2066,7 @@ function buildProgram4() {
2054
2066
  ).option("--write", "Apply promotions (write scripts and rewrite evals)").option("--provider <name>", "Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).action(
2055
2067
  async (paths, opts, command) => {
2056
2068
  try {
2057
- const pc = palette(colorFor2(command, process.stdout.isTTY));
2069
+ const pc = palette(colorFor(command, process.stdout.isTTY));
2058
2070
  const proposals = await runPromote(paths, {
2059
2071
  ...documentOptions(opts),
2060
2072
  write: opts.write,
@@ -2093,7 +2105,7 @@ function buildProgram4() {
2093
2105
  ).option("--no-cache", "Bypass the judge response cache").action(
2094
2106
  async (opts, command) => {
2095
2107
  try {
2096
- const color = colorFor2(command, process.stdout.isTTY);
2108
+ const color = colorFor(command, process.stdout.isTTY);
2097
2109
  const pc = palette(color);
2098
2110
  if (opts.seed) {
2099
2111
  const seeded = seedGoldenCases({ golden: opts.golden });
@@ -2148,7 +2160,7 @@ function buildProgram4() {
2148
2160
  if (!file) {
2149
2161
  console.log(
2150
2162
  renderReviews(listReviews(), {
2151
- color: colorFor2(command, process.stdout.isTTY)
2163
+ color: colorFor(command, process.stdout.isTTY)
2152
2164
  })
2153
2165
  );
2154
2166
  return;
@@ -6508,7 +6520,7 @@ async function runLint2(opts) {
6508
6520
 
6509
6521
  // src/term/commands/list.ts
6510
6522
  var LIST_FORMATS2 = ["pretty", "json", "csv"];
6511
- async function runList2(opts) {
6523
+ async function runList3(opts) {
6512
6524
  const { set } = await loadTerms(opts);
6513
6525
  return { terms: set.terms, set };
6514
6526
  }
@@ -7135,7 +7147,7 @@ function buildProgram8() {
7135
7147
  ).action(async (paths, options) => {
7136
7148
  try {
7137
7149
  const format = formatChoice(options.format, LIST_FORMATS2);
7138
- const { terms } = await runList2(await coreOptions(paths, options));
7150
+ const { terms } = await runList3(await coreOptions(paths, options));
7139
7151
  print(format === "json" ? renderListJson(terms) : format === "csv" ? renderListCsv(terms) : renderListPretty(terms));
7140
7152
  process.exitCode = 0;
7141
7153
  } catch (err) {
@@ -7236,13 +7248,376 @@ function buildProgram8() {
7236
7248
  return program;
7237
7249
  }
7238
7250
 
7239
- // src/cli.ts
7251
+ // src/tracevals/cli.ts
7252
+ import { Command as Command8 } from "commander";
7253
+ var { version } = package_default;
7240
7254
  function buildProgram9() {
7255
+ const program = new Command8();
7256
+ program.name("tracevals").description(
7257
+ "Deterministic and LLM-as-judge adherence evals for AI agent session traces."
7258
+ ).version(version).option("--no-color", "disable colored output").showHelpAfterError("(add --help for usage)").exitOverride();
7259
+ function collectGrant(value, previous) {
7260
+ if (!isExecutionGrant(value)) fail(new TracevalsError(allowExecutionMessage(value)));
7261
+ return [...previous, value];
7262
+ }
7263
+ function addConfigFlags(cmd) {
7264
+ return cmd.option("-c, --config <path>", "Path to manni.config.yaml").option("--no-config", "Ignore any discovered config file");
7265
+ }
7266
+ function configOptions(opts) {
7267
+ const { configPath, noConfig } = configOption(opts.config);
7268
+ return {
7269
+ ...configPath === void 0 ? {} : { config: configPath },
7270
+ ...noConfig === true ? { noConfig } : {}
7271
+ };
7272
+ }
7273
+ function parseFormatArg(name, allowed) {
7274
+ return (value) => {
7275
+ try {
7276
+ return parseFormat2(value, allowed, name);
7277
+ } catch (e) {
7278
+ fail(e);
7279
+ }
7280
+ };
7281
+ }
7282
+ const range = (min, max) => Number.isFinite(max) ? `between ${min} and ${max}` : `of at least ${min}`;
7283
+ function numeric(name, value, min, max = Number.POSITIVE_INFINITY) {
7284
+ if (value === void 0) return;
7285
+ if (Number.isFinite(value) && value >= min && value <= max) return;
7286
+ throw new TracevalsError(
7287
+ `${name} must be a number ${range(min, max)}, got ${value}`
7288
+ );
7289
+ }
7290
+ function whole(name, value, min, max = Number.POSITIVE_INFINITY) {
7291
+ if (value === void 0) return;
7292
+ if (Number.isInteger(value) && value >= min && value <= max) return;
7293
+ throw new TracevalsError(
7294
+ `${name} must be a whole number ${range(min, max)}, got ${value}`
7295
+ );
7296
+ }
7297
+ function sharedRunOptions(opts) {
7298
+ whole("--runs", opts.runs, 1);
7299
+ whole("--limit", opts.limit, 1);
7300
+ whole("--max-turns", opts.maxTurns, 1);
7301
+ return {
7302
+ ...configOptions(opts),
7303
+ project: opts.project,
7304
+ provider: opts.provider,
7305
+ model: opts.model,
7306
+ local: opts.local,
7307
+ // `undefined` rather than `false` when absent, so a future config
7308
+ // `offline:` still decides on a run where the flag was never typed.
7309
+ offline: opts.offline ? true : void 0,
7310
+ runs: opts.runs,
7311
+ deterministicOnly: opts.deterministicOnly,
7312
+ noCache: opts.cache === false,
7313
+ maxTurns: opts.maxTurns,
7314
+ format: opts.format ?? "pretty",
7315
+ output: opts.output,
7316
+ // Undefined when neither spelling is passed, so the config still decides.
7317
+ // Both spellings are declared for precisely that reason: commander
7318
+ // defaults a lone `--no-x` to `true`, which makes the config's own value
7319
+ // unreachable from the command line.
7320
+ failOnReview: opts.failOnReview,
7321
+ // Commander's default is `[]`, which is "no flag", not "run nothing": only
7322
+ // a list someone typed narrows the run, and `--no-execution` runs none.
7323
+ ...opts.execution === false ? { allowExecution: [] } : (opts.allowExecution ?? []).length > 0 ? { allowExecution: opts.allowExecution } : {},
7324
+ reportUnusedArtifacts: opts.reportUnusedArtifacts,
7325
+ ...opts.require !== void 0 ? { require: opts.require } : {},
7326
+ ...opts.allProjects !== void 0 ? { allProjects: opts.allProjects } : {},
7327
+ ...opts.newerThan !== void 0 ? { newerThan: opts.newerThan } : {},
7328
+ ...opts.limit !== void 0 ? { limit: opts.limit } : {}
7329
+ };
7330
+ }
7331
+ async function executeRun(traces, opts, color) {
7332
+ const shared = {
7333
+ ...sharedRunOptions(opts),
7334
+ color,
7335
+ // Run-only, both of them: `--history` is a point in one session's
7336
+ // timeline, and a manifest belongs to exactly one session (ADR 01024).
7337
+ history: opts.history,
7338
+ ...opts.manifest !== void 0 ? { manifest: opts.manifest } : {}
7339
+ };
7340
+ const selecting = opts.allProjects === true || opts.newerThan !== void 0 || opts.limit !== void 0;
7341
+ if (traces.length > 1 || selecting) {
7342
+ if (opts.manifest !== void 0) {
7343
+ throw new TracevalsError(
7344
+ "--manifest names one session's manifest, so it cannot be used with a corpus; drop it and each trace will find its own"
7345
+ );
7346
+ }
7347
+ const { runBatch } = await import("./batch-7JJWXWSF.js");
7348
+ const { report: report2, rendered: rendered3 } = await runBatch({
7349
+ ...shared,
7350
+ ...traces.length > 0 ? { traces } : {}
7351
+ });
7352
+ console.log(rendered3);
7353
+ process.exitCode = report2.exitCode;
7354
+ return;
7355
+ }
7356
+ let trace = traces[0];
7357
+ if (trace === void 0) {
7358
+ if (!process.stdout.isTTY || !process.stdin.isTTY) {
7359
+ throw new TracevalsError(
7360
+ "no trace given; pass a trace file or run `manni tracevals list`"
7361
+ );
7362
+ }
7363
+ const { pickTrace } = await import("./picker-6HRMETE7.js");
7364
+ trace = await pickTrace();
7365
+ }
7366
+ const { report, rendered: rendered2 } = await runRun2({ ...shared, tracePath: trace });
7367
+ console.log(rendered2);
7368
+ process.exitCode = report.exitCode;
7369
+ }
7370
+ function addRunFlags(cmd, options = {}) {
7371
+ const withHistory = options.history !== false;
7372
+ const formats = options.formats ?? REPORT_FORMATS3;
7373
+ const base = addConfigFlags(cmd).option(
7374
+ "--project <dir>",
7375
+ "artifact-lookup root (overrides the trace's recorded cwd)"
7376
+ ).option(
7377
+ "--provider <name>",
7378
+ "Judge provider: auto (default) | anthropic | openai | claude-cli | llama-cpp"
7379
+ ).option(
7380
+ "--model <model>",
7381
+ "Model override; needs a named provider, from here or config"
7382
+ ).option("--local", LOCAL_FLAG_HELP).option("--runs <n>", "ensemble runs per eval", (v) => Number(v)).option("--deterministic-only", "skip LLM-judged evals").option(
7383
+ "--offline",
7384
+ "never fetch a remote manifest; read external metadata from local manifests"
7385
+ ).option("--no-cache", "bypass the judge cache").option(
7386
+ "--max-turns <n>",
7387
+ "Stop after this many ensemble runs (a cached ensemble costs none)",
7388
+ (v) => Number(v)
7389
+ ).option(
7390
+ "-f, --format <format>",
7391
+ `Output format: ${formats.join(" | ")}`,
7392
+ parseFormatArg("--format", formats),
7393
+ "pretty"
7394
+ ).option("-o, --output <file>", "also write the report to a file").option(
7395
+ "--report-unused-artifacts",
7396
+ "list every skill and agent the session was offered and never used"
7397
+ ).option("--fail-on-review", "Exit 1 when any eval lands in the human-review zone (the default)").option("--no-fail-on-review", "Do not fail the run on evals left for human review").option(
7398
+ "--allow-execution <kind>",
7399
+ "Run only these execution grants (repeatable): frontmatter-commands",
7400
+ collectGrant,
7401
+ []
7402
+ ).option("--no-execution", "Clear every execution grant for this run").option(
7403
+ "--require <module>",
7404
+ "load a grader plugin; repeatable, and added to config plugins",
7405
+ collect,
7406
+ []
7407
+ ).option("--all-projects", "evaluate every project's traces in the session store").option(
7408
+ "--newer-than <duration>",
7409
+ "only traces modified within this window, such as 30m, 24h, 7d or 2w"
7410
+ ).option("--limit <n>", "maximum traces to evaluate", (v) => Number(v));
7411
+ return withHistory ? base.option(
7412
+ "--history",
7413
+ "append to history and compare with the previous run"
7414
+ ) : base;
7415
+ }
7416
+ addRunFlags(
7417
+ // No `isDefault` (proposal 0034): `run` is a verb, so a bare
7418
+ // `manni tracevals` prints the help and exits 2, and a trace path with no
7419
+ // verb is commander's unknown-command refusal rather than a silent run.
7420
+ program.command("run [traces...]").description(
7421
+ "Evaluate one or more traces against the skills and instructions they used"
7422
+ )
7423
+ ).option(
7424
+ "--manifest <file>",
7425
+ "session manifest to compare artifacts against; without it one is looked for beside the trace and under the project"
7426
+ ).action(async (traces, opts, command) => {
7427
+ try {
7428
+ await executeRun(traces, opts, colorFor(command, process.stdout.isTTY));
7429
+ } catch (e) {
7430
+ fail(e);
7431
+ }
7432
+ });
7433
+ addRunFlags(
7434
+ program.command("calibrate [traces...]").description(
7435
+ "Measure judged and graded verdicts against a labels file: false passes, false fails, and review volume"
7436
+ ).option(
7437
+ "--labels <file>",
7438
+ "calibration labels sidecar (default: calibrate.labels)"
7439
+ ).option(
7440
+ "--sweep",
7441
+ "re-score the corpus across the configured grid of zones and ensemble sizes, from cached verdicts"
7442
+ ).option(
7443
+ "--max-false-pass <n>",
7444
+ "exit 1 above this many false passes",
7445
+ (v) => Number(v)
7446
+ ).option(
7447
+ "--max-false-fail <n>",
7448
+ "exit 1 above this many false fails",
7449
+ (v) => Number(v)
7450
+ ).option(
7451
+ "--max-review <n>",
7452
+ "exit 1 above this many needs-review outcomes",
7453
+ (v) => Number(v)
7454
+ ),
7455
+ { history: false, formats: CALIBRATE_FORMATS }
7456
+ ).action(
7457
+ async (traces, opts, command) => {
7458
+ try {
7459
+ const shared = {
7460
+ ...sharedRunOptions(opts),
7461
+ color: colorFor(command, process.stdout.isTTY)
7462
+ };
7463
+ whole("--max-false-pass", opts.maxFalsePass, 0);
7464
+ whole("--max-false-fail", opts.maxFalseFail, 0);
7465
+ whole("--max-review", opts.maxReview, 0);
7466
+ const { runCalibrate: runCalibrate2 } = await import("./calibrate-IA2ARFMH.js");
7467
+ const { report, rendered: rendered2 } = await runCalibrate2({
7468
+ ...shared,
7469
+ ...traces.length > 0 ? { traces } : {},
7470
+ ...opts.labels !== void 0 ? { labels: opts.labels } : {},
7471
+ ...opts.sweep !== void 0 ? { sweep: opts.sweep } : {},
7472
+ ...opts.maxFalsePass !== void 0 ? { maxFalsePass: opts.maxFalsePass } : {},
7473
+ ...opts.maxFalseFail !== void 0 ? { maxFalseFail: opts.maxFalseFail } : {},
7474
+ ...opts.maxReview !== void 0 ? { maxReview: opts.maxReview } : {}
7475
+ });
7476
+ console.log(rendered2);
7477
+ process.exitCode = report.exitCode;
7478
+ } catch (e) {
7479
+ fail(e);
7480
+ }
7481
+ }
7482
+ );
7483
+ addConfigFlags(
7484
+ program.command("fill [paths...]").description(
7485
+ "Propose evals for skills, agent definitions, and project rules, and write those above the confidence threshold"
7486
+ )
7487
+ ).option("--project <dir>", "project root to scan (default: current directory)").option("--exclude <glob>", "Glob to exclude (repeatable)", collect, []).option("--dry-run", "report proposals without writing them").option(
7488
+ "--confidence <n>",
7489
+ "minimum confidence to write (0-1)",
7490
+ (v) => parseFloat(v)
7491
+ ).option(
7492
+ "--max-evals <n>",
7493
+ "maximum evals per artifact, including existing ones",
7494
+ (v) => Number(v)
7495
+ ).option(
7496
+ "--max-turns <n>",
7497
+ "Stop after this many inference calls (a cached artifact costs none)",
7498
+ (v) => Number(v)
7499
+ ).option("--no-cache", "bypass the proposal cache").option(
7500
+ "--provider <name>",
7501
+ "Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp"
7502
+ ).option(
7503
+ "--model <model>",
7504
+ "Model override; needs a named provider, from here or config"
7505
+ ).option("--local", LOCAL_FLAG_HELP).option(
7506
+ "--offline",
7507
+ "never fetch a remote manifest; read external metadata from local manifests"
7508
+ ).option(
7509
+ "--require <module>",
7510
+ "load a grader plugin; repeatable, and added to config plugins",
7511
+ collect,
7512
+ []
7513
+ ).option(
7514
+ "-f, --format <format>",
7515
+ `Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
7516
+ parseFormatArg("--format", SUMMARY_FORMATS2),
7517
+ "pretty"
7518
+ ).action(async (paths, opts, command) => {
7519
+ try {
7520
+ numeric("--confidence", opts.confidence, 0, 1);
7521
+ whole("--max-evals", opts.maxEvals, 1);
7522
+ whole("--max-turns", opts.maxTurns, 1);
7523
+ const { report, rendered: rendered2 } = await runFill3({
7524
+ ...configOptions(opts),
7525
+ ...paths.length > 0 ? { paths } : {},
7526
+ ...opts.project !== void 0 ? { project: opts.project } : {},
7527
+ ...opts.exclude !== void 0 ? { exclude: opts.exclude } : {},
7528
+ ...opts.dryRun !== void 0 ? { dryRun: opts.dryRun } : {},
7529
+ ...opts.confidence !== void 0 ? { confidence: opts.confidence } : {},
7530
+ ...opts.maxEvals !== void 0 ? { maxEvals: opts.maxEvals } : {},
7531
+ ...opts.maxTurns !== void 0 ? { maxTurns: opts.maxTurns } : {},
7532
+ ...opts.cache === false ? { noCache: true } : {},
7533
+ ...opts.provider !== void 0 ? { provider: opts.provider } : {},
7534
+ ...opts.model !== void 0 ? { model: opts.model } : {},
7535
+ ...opts.local !== void 0 ? { local: opts.local } : {},
7536
+ ...opts.offline === true ? { offline: true } : {},
7537
+ ...opts.require !== void 0 ? { require: opts.require } : {},
7538
+ // P1 (proposal 0047): offered only on a terminal, where there is
7539
+ // someone to answer. Off one the block goes to the artifact and the
7540
+ // W1 line says so.
7541
+ confirm: terminalConfirm(),
7542
+ onNotice: notice,
7543
+ onRelocated: (result) => {
7544
+ const manifests = result.manifests.map((m) => m.file).join(", ");
7545
+ notice(`moved ${String(result.summary.moved)} value(s) into ${manifests}.`);
7546
+ },
7547
+ color: colorFor(command, process.stdout.isTTY)
7548
+ });
7549
+ console.log(
7550
+ opts.format === "json" ? JSON.stringify(report, null, 2) : rendered2
7551
+ );
7552
+ process.exitCode = report.exitCode;
7553
+ } catch (e) {
7554
+ fail(e);
7555
+ }
7556
+ });
7557
+ addConfigFlags(
7558
+ program.command("capture").description(
7559
+ "Record a session manifest: sha256 of every instruction artifact plus the git SHA, read from a Claude Code hook payload on stdin"
7560
+ )
7561
+ ).option("--project <dir>", "project root to scan (default: the payload's cwd)").option("--session-id <id>", "session id (default: the payload's session_id)").option("-o, --out <file>", "write here instead of the configured directory").option(
7562
+ "-f, --format <format>",
7563
+ `Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
7564
+ parseFormatArg("--format", SUMMARY_FORMATS2),
7565
+ "pretty"
7566
+ ).action(async (opts) => {
7567
+ try {
7568
+ const { runCapture } = await import("./capture-C32JXIFN.js");
7569
+ const result = await runCapture({
7570
+ version,
7571
+ ...configOptions(opts),
7572
+ ...opts.project !== void 0 ? { project: opts.project } : {},
7573
+ ...opts.sessionId !== void 0 ? { sessionId: opts.sessionId } : {},
7574
+ ...opts.out !== void 0 ? { out: opts.out } : {},
7575
+ format: opts.format ?? "pretty"
7576
+ });
7577
+ if (result.stdout !== "") console.log(result.stdout);
7578
+ if (result.stderr !== "") notice(result.stderr);
7579
+ process.exitCode = result.exitCode;
7580
+ } catch (e) {
7581
+ fail(e);
7582
+ }
7583
+ });
7584
+ program.command("list").description("List discoverable traces (Claude Code session files)").option(
7585
+ "--project <dir>",
7586
+ "project directory to scope to (default: current directory)"
7587
+ ).option("--all-projects", "scan every project in the session store").option(
7588
+ "--newer-than <duration>",
7589
+ "only traces modified within this window, such as 30m, 24h, 7d or 2w"
7590
+ ).option("--limit <n>", "maximum traces to list", (v) => Number(v)).option(
7591
+ "-f, --format <format>",
7592
+ `Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
7593
+ parseFormatArg("--format", SUMMARY_FORMATS2),
7594
+ "pretty"
7595
+ ).action(async (opts, command) => {
7596
+ try {
7597
+ whole("--limit", opts.limit, 1);
7598
+ const run = await runList2({
7599
+ project: opts.project,
7600
+ allProjects: opts.allProjects,
7601
+ ...opts.newerThan !== void 0 ? { newerThan: opts.newerThan } : {},
7602
+ limit: opts.limit
7603
+ });
7604
+ console.log(
7605
+ opts.format === "json" ? JSON.stringify(run, null, 2) : renderList2(run, { color: colorFor(command, process.stdout.isTTY) })
7606
+ );
7607
+ } catch (e) {
7608
+ fail(e);
7609
+ }
7610
+ });
7611
+ return program;
7612
+ }
7613
+
7614
+ // src/cli.ts
7615
+ function buildProgram10() {
7241
7616
  const meta = buildProgram().name("meta").description(
7242
7617
  "Validate, read, query and fill document metadata against JSON Schema"
7243
7618
  );
7244
7619
  const metaCommands = new Set(meta.commands.map((c) => c.name()));
7245
- const program = new Command8();
7620
+ const program = new Command9();
7246
7621
  program.name("manni").description("Tools for documentation that is meant to be checked.").version(package_default.version, "-V, --version").showHelpAfterError("(add --help for usage)").exitOverride().configureOutput({
7247
7622
  outputError: (str, write) => {
7248
7623
  write(str);
@@ -7285,13 +7660,18 @@ function buildProgram9() {
7285
7660
  "Deterministic knowledge graphs derived from documentation frontmatter and formatting"
7286
7661
  )
7287
7662
  );
7663
+ program.addCommand(
7664
+ buildProgram9().name("tracevals").description(
7665
+ "Deterministic and LLM-as-judge adherence evals for AI agent session traces"
7666
+ )
7667
+ );
7288
7668
  program.addCommand(
7289
7669
  buildProgram5().name("key").description("Set and rotate the family key that encrypted values are encrypted with.")
7290
7670
  );
7291
7671
  return program;
7292
7672
  }
7293
- runIfMain(import.meta.url, buildProgram9);
7673
+ runIfMain(import.meta.url, buildProgram10);
7294
7674
  export {
7295
- buildProgram9 as buildProgram
7675
+ buildProgram10 as buildProgram
7296
7676
  };
7297
7677
  //# sourceMappingURL=cli.js.map