@hawkeyexl/manni 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/batch-7JJWXWSF.js +18 -0
- package/dist/batch-7JJWXWSF.js.map +1 -0
- package/dist/calibrate-IA2ARFMH.js +13 -0
- package/dist/calibrate-IA2ARFMH.js.map +1 -0
- package/dist/capture-C32JXIFN.js +15 -0
- package/dist/capture-C32JXIFN.js.map +1 -0
- package/dist/chunk-3R23C5DT.js +120 -0
- package/dist/chunk-3R23C5DT.js.map +1 -0
- package/dist/chunk-CI2GOU3A.js +4447 -0
- package/dist/chunk-CI2GOU3A.js.map +1 -0
- package/dist/{chunk-TH565O2N.js → chunk-DP2ZA3WH.js} +898 -278
- package/dist/chunk-DP2ZA3WH.js.map +1 -0
- package/dist/chunk-ESJ5W7EV.js +297 -0
- package/dist/chunk-ESJ5W7EV.js.map +1 -0
- package/dist/chunk-IOYXPBCL.js +37 -0
- package/dist/chunk-IOYXPBCL.js.map +1 -0
- package/dist/chunk-KTSJ2ZHA.js +40 -0
- package/dist/chunk-KTSJ2ZHA.js.map +1 -0
- package/dist/{chunk-BNHQWIAE.js → chunk-MFDD3YGU.js} +2 -2
- package/dist/chunk-MOQXWXSG.js +1941 -0
- package/dist/chunk-MOQXWXSG.js.map +1 -0
- package/dist/{chunk-UYDRDFYF.js → chunk-O6RVQKK6.js} +1943 -909
- package/dist/chunk-O6RVQKK6.js.map +1 -0
- package/dist/chunk-QROCFW5S.js +174 -0
- package/dist/chunk-QROCFW5S.js.map +1 -0
- package/dist/chunk-RVVCA4WL.js +379 -0
- package/dist/chunk-RVVCA4WL.js.map +1 -0
- package/dist/cli.js +422 -42
- package/dist/cli.js.map +1 -1
- package/dist/docmeta.js +3 -2
- package/dist/docmeta.js.map +1 -1
- package/dist/index.d.ts +3873 -795
- package/dist/index.js +292 -4
- package/dist/index.js.map +1 -1
- package/dist/picker-6HRMETE7.js +12 -0
- package/dist/picker-6HRMETE7.js.map +1 -0
- package/package.json +3 -1
- package/plugin/tracevals/.claude-plugin/plugin.json +9 -0
- package/plugin/tracevals/hooks/hooks.json +17 -0
- package/dist/chunk-TH565O2N.js.map +0 -1
- package/dist/chunk-UYDRDFYF.js.map +0 -1
- /package/dist/{chunk-BNHQWIAE.js.map → chunk-MFDD3YGU.js.map} +0 -0
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
buildProgram
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-MFDD3YGU.js";
|
|
5
5
|
import {
|
|
6
6
|
ALL_FILL_FIELDS,
|
|
7
7
|
CONFIG_SECTION,
|
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
DEFAULT_CONFIG_FILENAME2 as DEFAULT_CONFIG_FILENAME,
|
|
10
10
|
DEFAULT_EXTENSIONS,
|
|
11
11
|
DocevalsError,
|
|
12
|
-
EXECUTION_GRANTS,
|
|
13
12
|
EXPORT_TARGETS,
|
|
14
13
|
GraphError,
|
|
15
14
|
INFER_FORMATS,
|
|
@@ -52,6 +51,7 @@ import {
|
|
|
52
51
|
renderFill,
|
|
53
52
|
renderFill2,
|
|
54
53
|
renderList,
|
|
54
|
+
renderList2,
|
|
55
55
|
renderQuery,
|
|
56
56
|
renderRemoveJson,
|
|
57
57
|
renderRemovePretty,
|
|
@@ -72,12 +72,14 @@ import {
|
|
|
72
72
|
runExport,
|
|
73
73
|
runFill,
|
|
74
74
|
runFill2,
|
|
75
|
+
runFill3,
|
|
75
76
|
runGenerate,
|
|
76
77
|
runInit,
|
|
77
78
|
runKeyRotate,
|
|
78
79
|
runKeySet,
|
|
79
80
|
runLint,
|
|
80
81
|
runList,
|
|
82
|
+
runList2,
|
|
81
83
|
runPromote,
|
|
82
84
|
runQuery,
|
|
83
85
|
runRemove,
|
|
@@ -97,7 +99,19 @@ import {
|
|
|
97
99
|
shortSrc,
|
|
98
100
|
spellSource,
|
|
99
101
|
validateGraph
|
|
100
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-O6RVQKK6.js";
|
|
103
|
+
import {
|
|
104
|
+
CALIBRATE_FORMATS,
|
|
105
|
+
REPORT_FORMATS as REPORT_FORMATS3,
|
|
106
|
+
SUMMARY_FORMATS as SUMMARY_FORMATS2,
|
|
107
|
+
parseFormat as parseFormat2,
|
|
108
|
+
runRun as runRun2
|
|
109
|
+
} from "./chunk-CI2GOU3A.js";
|
|
110
|
+
import "./chunk-QROCFW5S.js";
|
|
111
|
+
import {
|
|
112
|
+
allowExecutionMessage,
|
|
113
|
+
isExecutionGrant
|
|
114
|
+
} from "./chunk-MOQXWXSG.js";
|
|
101
115
|
import {
|
|
102
116
|
COMMON_FORMAT_LIST,
|
|
103
117
|
CONCEPT_FIELDS,
|
|
@@ -109,11 +123,10 @@ import {
|
|
|
109
123
|
STDIN_LABEL,
|
|
110
124
|
STDIN_TOKEN,
|
|
111
125
|
TERM_RELATION_FIELDS,
|
|
112
|
-
ToolError,
|
|
113
126
|
assertNonEmpty,
|
|
114
127
|
collect,
|
|
128
|
+
colorFor,
|
|
115
129
|
configOption,
|
|
116
|
-
errorMessage,
|
|
117
130
|
escapeWorkflowCommandMessage,
|
|
118
131
|
escapeWorkflowCommandProperty,
|
|
119
132
|
explicitFalse,
|
|
@@ -151,10 +164,17 @@ import {
|
|
|
151
164
|
valueAt,
|
|
152
165
|
warn,
|
|
153
166
|
writeTextAtomic
|
|
154
|
-
} from "./chunk-
|
|
167
|
+
} from "./chunk-DP2ZA3WH.js";
|
|
168
|
+
import {
|
|
169
|
+
TracevalsError
|
|
170
|
+
} from "./chunk-3R23C5DT.js";
|
|
171
|
+
import {
|
|
172
|
+
ToolError,
|
|
173
|
+
errorMessage
|
|
174
|
+
} from "./chunk-IOYXPBCL.js";
|
|
155
175
|
|
|
156
176
|
// src/cli.ts
|
|
157
|
-
import { Command as
|
|
177
|
+
import { Command as Command9 } from "commander";
|
|
158
178
|
|
|
159
179
|
// src/a11y/cli.ts
|
|
160
180
|
import { Command } from "commander";
|
|
@@ -1375,7 +1395,7 @@ function renderCheckJunit(run) {
|
|
|
1375
1395
|
// src/cite/cli.ts
|
|
1376
1396
|
var UPDATE_FORMATS = ["pretty", "json"];
|
|
1377
1397
|
var REMOVE_FORMATS = UPDATE_FORMATS;
|
|
1378
|
-
function
|
|
1398
|
+
function colorFor2(command, isTTY, env) {
|
|
1379
1399
|
const noColor = colorOwner(command).opts().color === false;
|
|
1380
1400
|
return shouldColor({ noColor, isTTY, env });
|
|
1381
1401
|
}
|
|
@@ -1502,7 +1522,7 @@ function buildProgram3() {
|
|
|
1502
1522
|
switch (format) {
|
|
1503
1523
|
case "pretty":
|
|
1504
1524
|
text2 = renderCheckPretty(run, {
|
|
1505
|
-
color:
|
|
1525
|
+
color: colorFor2(command, process.stdout.isTTY),
|
|
1506
1526
|
quiet: Boolean(options.quiet),
|
|
1507
1527
|
showDiff: Boolean(options.showDiff),
|
|
1508
1528
|
reveal: Boolean(options.reveal)
|
|
@@ -1666,7 +1686,7 @@ function buildProgram3() {
|
|
|
1666
1686
|
dryRun
|
|
1667
1687
|
});
|
|
1668
1688
|
const text2 = format === "json" ? renderUpdateJson(run) : renderUpdatePretty(run, {
|
|
1669
|
-
color:
|
|
1689
|
+
color: colorFor2(command, process.stdout.isTTY),
|
|
1670
1690
|
// The diffs are the point of a dry run; a real run prints
|
|
1671
1691
|
// what it rewrote and leaves the file to say the rest.
|
|
1672
1692
|
showDiff: dryRun
|
|
@@ -1741,7 +1761,7 @@ function buildProgram3() {
|
|
|
1741
1761
|
dryRun
|
|
1742
1762
|
});
|
|
1743
1763
|
const text2 = format === "json" ? renderRemoveJson(run) : renderRemovePretty(run, {
|
|
1744
|
-
color:
|
|
1764
|
+
color: colorFor2(command, process.stdout.isTTY),
|
|
1745
1765
|
// The diffs are the point of a dry run; a real run prints
|
|
1746
1766
|
// what it removed and leaves the file to say the rest.
|
|
1747
1767
|
showDiff: dryRun,
|
|
@@ -1766,28 +1786,14 @@ function buildProgram3() {
|
|
|
1766
1786
|
|
|
1767
1787
|
// src/docevals/cli.ts
|
|
1768
1788
|
import { Command as Command3 } from "commander";
|
|
1769
|
-
function colorFor2(command, isTTY, env) {
|
|
1770
|
-
const noColor = colorOwner2(command).opts().color === false;
|
|
1771
|
-
return shouldColor({ noColor, isTTY, env });
|
|
1772
|
-
}
|
|
1773
|
-
function colorOwner2(command) {
|
|
1774
|
-
for (let c = command; c !== null; c = c.parent) {
|
|
1775
|
-
if (c.options.some((o) => o.long === "--no-color")) return c;
|
|
1776
|
-
}
|
|
1777
|
-
return command;
|
|
1778
|
-
}
|
|
1779
1789
|
function buildProgram4() {
|
|
1780
1790
|
const program = new Command3();
|
|
1781
1791
|
program.name("docevals").description(
|
|
1782
1792
|
"Deterministic and LLM-as-judge evals for documentation pages, driven by frontmatter."
|
|
1783
1793
|
).version(package_default.version).option("--no-color", "disable colored output").showHelpAfterError("(add --help for usage)").exitOverride();
|
|
1784
1794
|
function collectGrant(value, previous) {
|
|
1785
|
-
if (!
|
|
1786
|
-
fail(
|
|
1787
|
-
new DocevalsError(
|
|
1788
|
-
`--allow-execution must be one of ${EXECUTION_GRANTS.join(" | ")}, got "${value}"`
|
|
1789
|
-
)
|
|
1790
|
-
);
|
|
1795
|
+
if (!isExecutionGrant(value)) {
|
|
1796
|
+
fail(new DocevalsError(allowExecutionMessage(value)));
|
|
1791
1797
|
}
|
|
1792
1798
|
return [...previous, value];
|
|
1793
1799
|
}
|
|
@@ -1860,7 +1866,7 @@ function buildProgram4() {
|
|
|
1860
1866
|
});
|
|
1861
1867
|
console.log(
|
|
1862
1868
|
renderList(run, opts.format, {
|
|
1863
|
-
color:
|
|
1869
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
1864
1870
|
})
|
|
1865
1871
|
);
|
|
1866
1872
|
process.exitCode = run.exitCode;
|
|
@@ -1883,7 +1889,7 @@ function buildProgram4() {
|
|
|
1883
1889
|
// frontmatter-commands docs/**` would silently swallow the glob and run
|
|
1884
1890
|
// over the default file set instead. Repeat the flag to grant twice.
|
|
1885
1891
|
"--allow-execution <kind>",
|
|
1886
|
-
"
|
|
1892
|
+
"Run only these execution grants (repeatable): frontmatter-commands",
|
|
1887
1893
|
collectGrant,
|
|
1888
1894
|
[]
|
|
1889
1895
|
).option("--no-execution", "Clear every execution grant for this run").option("--no-generate", "Do not generate scripts for command evals missing a command").option("--no-cache", "Bypass the judge response cache").option("--fail-on-review", "Exit 1 when any eval lands in the human-review zone").option("--provider <name>", "Judge provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).option("--runs <n>", "Ensemble runs per eval", parseIntArg("--runs")).option(
|
|
@@ -1898,6 +1904,9 @@ function buildProgram4() {
|
|
|
1898
1904
|
).option("--suite <name>", "Run only evals in this suite").option(
|
|
1899
1905
|
"--since <ref>",
|
|
1900
1906
|
"Evaluate only pages whose file or eval manifest changed between this git ref and HEAD"
|
|
1907
|
+
).option(
|
|
1908
|
+
"--newer-than <duration>",
|
|
1909
|
+
"Evaluate only pages whose file or eval manifest changed within this window, such as 30m, 24h, 7d or 2w; in CI, prefer --since"
|
|
1901
1910
|
).option(
|
|
1902
1911
|
"--max-turns <n>",
|
|
1903
1912
|
"Stop after this many ensemble runs (a cached ensemble costs none)",
|
|
@@ -1921,7 +1930,9 @@ function buildProgram4() {
|
|
|
1921
1930
|
format: opts.format,
|
|
1922
1931
|
deterministicOnly: opts.deterministicOnly,
|
|
1923
1932
|
aiOnly: opts.aiOnly,
|
|
1924
|
-
|
|
1933
|
+
// Commander's default is `[]`, which is "no flag", not "run nothing":
|
|
1934
|
+
// only a list someone typed narrows the run.
|
|
1935
|
+
allowExecution: opts.allowExecution.length > 0 ? opts.allowExecution : void 0,
|
|
1925
1936
|
execution: opts.execution,
|
|
1926
1937
|
generate: opts.generate,
|
|
1927
1938
|
cache: opts.cache,
|
|
@@ -1935,6 +1946,7 @@ function buildProgram4() {
|
|
|
1935
1946
|
evalNames: opts.eval,
|
|
1936
1947
|
suite: opts.suite,
|
|
1937
1948
|
since: opts.since,
|
|
1949
|
+
newerThan: opts.newerThan,
|
|
1938
1950
|
// commander collapses `--baseline` to true and `--no-baseline` to
|
|
1939
1951
|
// false on the same key; a string is an explicit path.
|
|
1940
1952
|
baseline: opts.baseline,
|
|
@@ -1943,7 +1955,7 @@ function buildProgram4() {
|
|
|
1943
1955
|
});
|
|
1944
1956
|
console.log(
|
|
1945
1957
|
render(report, opts.format, {
|
|
1946
|
-
color:
|
|
1958
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
1947
1959
|
})
|
|
1948
1960
|
);
|
|
1949
1961
|
process.exitCode = report.exitCode;
|
|
@@ -1959,7 +1971,7 @@ function buildProgram4() {
|
|
|
1959
1971
|
).option("--provider <name>", "Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).action(
|
|
1960
1972
|
async (paths, opts, command) => {
|
|
1961
1973
|
try {
|
|
1962
|
-
const pc = palette(
|
|
1974
|
+
const pc = palette(colorFor(command, process.stdout.isTTY));
|
|
1963
1975
|
const result = await runGenerate(paths, {
|
|
1964
1976
|
...documentOptions(opts),
|
|
1965
1977
|
provider: opts.provider,
|
|
@@ -2038,7 +2050,7 @@ function buildProgram4() {
|
|
|
2038
2050
|
});
|
|
2039
2051
|
console.log(
|
|
2040
2052
|
renderFill(report, opts.format, {
|
|
2041
|
-
color:
|
|
2053
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
2042
2054
|
})
|
|
2043
2055
|
);
|
|
2044
2056
|
process.exitCode = report.exitCode;
|
|
@@ -2054,7 +2066,7 @@ function buildProgram4() {
|
|
|
2054
2066
|
).option("--write", "Apply promotions (write scripts and rewrite evals)").option("--provider <name>", "Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp").option("--model <model>", "Model override; needs a named provider, from here or config").option("--local", LOCAL_FLAG_HELP).action(
|
|
2055
2067
|
async (paths, opts, command) => {
|
|
2056
2068
|
try {
|
|
2057
|
-
const pc = palette(
|
|
2069
|
+
const pc = palette(colorFor(command, process.stdout.isTTY));
|
|
2058
2070
|
const proposals = await runPromote(paths, {
|
|
2059
2071
|
...documentOptions(opts),
|
|
2060
2072
|
write: opts.write,
|
|
@@ -2093,7 +2105,7 @@ function buildProgram4() {
|
|
|
2093
2105
|
).option("--no-cache", "Bypass the judge response cache").action(
|
|
2094
2106
|
async (opts, command) => {
|
|
2095
2107
|
try {
|
|
2096
|
-
const color =
|
|
2108
|
+
const color = colorFor(command, process.stdout.isTTY);
|
|
2097
2109
|
const pc = palette(color);
|
|
2098
2110
|
if (opts.seed) {
|
|
2099
2111
|
const seeded = seedGoldenCases({ golden: opts.golden });
|
|
@@ -2148,7 +2160,7 @@ function buildProgram4() {
|
|
|
2148
2160
|
if (!file) {
|
|
2149
2161
|
console.log(
|
|
2150
2162
|
renderReviews(listReviews(), {
|
|
2151
|
-
color:
|
|
2163
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
2152
2164
|
})
|
|
2153
2165
|
);
|
|
2154
2166
|
return;
|
|
@@ -6508,7 +6520,7 @@ async function runLint2(opts) {
|
|
|
6508
6520
|
|
|
6509
6521
|
// src/term/commands/list.ts
|
|
6510
6522
|
var LIST_FORMATS2 = ["pretty", "json", "csv"];
|
|
6511
|
-
async function
|
|
6523
|
+
async function runList3(opts) {
|
|
6512
6524
|
const { set } = await loadTerms(opts);
|
|
6513
6525
|
return { terms: set.terms, set };
|
|
6514
6526
|
}
|
|
@@ -7135,7 +7147,7 @@ function buildProgram8() {
|
|
|
7135
7147
|
).action(async (paths, options) => {
|
|
7136
7148
|
try {
|
|
7137
7149
|
const format = formatChoice(options.format, LIST_FORMATS2);
|
|
7138
|
-
const { terms } = await
|
|
7150
|
+
const { terms } = await runList3(await coreOptions(paths, options));
|
|
7139
7151
|
print(format === "json" ? renderListJson(terms) : format === "csv" ? renderListCsv(terms) : renderListPretty(terms));
|
|
7140
7152
|
process.exitCode = 0;
|
|
7141
7153
|
} catch (err) {
|
|
@@ -7236,13 +7248,376 @@ function buildProgram8() {
|
|
|
7236
7248
|
return program;
|
|
7237
7249
|
}
|
|
7238
7250
|
|
|
7239
|
-
// src/cli.ts
|
|
7251
|
+
// src/tracevals/cli.ts
|
|
7252
|
+
import { Command as Command8 } from "commander";
|
|
7253
|
+
var { version } = package_default;
|
|
7240
7254
|
function buildProgram9() {
|
|
7255
|
+
const program = new Command8();
|
|
7256
|
+
program.name("tracevals").description(
|
|
7257
|
+
"Deterministic and LLM-as-judge adherence evals for AI agent session traces."
|
|
7258
|
+
).version(version).option("--no-color", "disable colored output").showHelpAfterError("(add --help for usage)").exitOverride();
|
|
7259
|
+
function collectGrant(value, previous) {
|
|
7260
|
+
if (!isExecutionGrant(value)) fail(new TracevalsError(allowExecutionMessage(value)));
|
|
7261
|
+
return [...previous, value];
|
|
7262
|
+
}
|
|
7263
|
+
function addConfigFlags(cmd) {
|
|
7264
|
+
return cmd.option("-c, --config <path>", "Path to manni.config.yaml").option("--no-config", "Ignore any discovered config file");
|
|
7265
|
+
}
|
|
7266
|
+
function configOptions(opts) {
|
|
7267
|
+
const { configPath, noConfig } = configOption(opts.config);
|
|
7268
|
+
return {
|
|
7269
|
+
...configPath === void 0 ? {} : { config: configPath },
|
|
7270
|
+
...noConfig === true ? { noConfig } : {}
|
|
7271
|
+
};
|
|
7272
|
+
}
|
|
7273
|
+
function parseFormatArg(name, allowed) {
|
|
7274
|
+
return (value) => {
|
|
7275
|
+
try {
|
|
7276
|
+
return parseFormat2(value, allowed, name);
|
|
7277
|
+
} catch (e) {
|
|
7278
|
+
fail(e);
|
|
7279
|
+
}
|
|
7280
|
+
};
|
|
7281
|
+
}
|
|
7282
|
+
const range = (min, max) => Number.isFinite(max) ? `between ${min} and ${max}` : `of at least ${min}`;
|
|
7283
|
+
function numeric(name, value, min, max = Number.POSITIVE_INFINITY) {
|
|
7284
|
+
if (value === void 0) return;
|
|
7285
|
+
if (Number.isFinite(value) && value >= min && value <= max) return;
|
|
7286
|
+
throw new TracevalsError(
|
|
7287
|
+
`${name} must be a number ${range(min, max)}, got ${value}`
|
|
7288
|
+
);
|
|
7289
|
+
}
|
|
7290
|
+
function whole(name, value, min, max = Number.POSITIVE_INFINITY) {
|
|
7291
|
+
if (value === void 0) return;
|
|
7292
|
+
if (Number.isInteger(value) && value >= min && value <= max) return;
|
|
7293
|
+
throw new TracevalsError(
|
|
7294
|
+
`${name} must be a whole number ${range(min, max)}, got ${value}`
|
|
7295
|
+
);
|
|
7296
|
+
}
|
|
7297
|
+
function sharedRunOptions(opts) {
|
|
7298
|
+
whole("--runs", opts.runs, 1);
|
|
7299
|
+
whole("--limit", opts.limit, 1);
|
|
7300
|
+
whole("--max-turns", opts.maxTurns, 1);
|
|
7301
|
+
return {
|
|
7302
|
+
...configOptions(opts),
|
|
7303
|
+
project: opts.project,
|
|
7304
|
+
provider: opts.provider,
|
|
7305
|
+
model: opts.model,
|
|
7306
|
+
local: opts.local,
|
|
7307
|
+
// `undefined` rather than `false` when absent, so a future config
|
|
7308
|
+
// `offline:` still decides on a run where the flag was never typed.
|
|
7309
|
+
offline: opts.offline ? true : void 0,
|
|
7310
|
+
runs: opts.runs,
|
|
7311
|
+
deterministicOnly: opts.deterministicOnly,
|
|
7312
|
+
noCache: opts.cache === false,
|
|
7313
|
+
maxTurns: opts.maxTurns,
|
|
7314
|
+
format: opts.format ?? "pretty",
|
|
7315
|
+
output: opts.output,
|
|
7316
|
+
// Undefined when neither spelling is passed, so the config still decides.
|
|
7317
|
+
// Both spellings are declared for precisely that reason: commander
|
|
7318
|
+
// defaults a lone `--no-x` to `true`, which makes the config's own value
|
|
7319
|
+
// unreachable from the command line.
|
|
7320
|
+
failOnReview: opts.failOnReview,
|
|
7321
|
+
// Commander's default is `[]`, which is "no flag", not "run nothing": only
|
|
7322
|
+
// a list someone typed narrows the run, and `--no-execution` runs none.
|
|
7323
|
+
...opts.execution === false ? { allowExecution: [] } : (opts.allowExecution ?? []).length > 0 ? { allowExecution: opts.allowExecution } : {},
|
|
7324
|
+
reportUnusedArtifacts: opts.reportUnusedArtifacts,
|
|
7325
|
+
...opts.require !== void 0 ? { require: opts.require } : {},
|
|
7326
|
+
...opts.allProjects !== void 0 ? { allProjects: opts.allProjects } : {},
|
|
7327
|
+
...opts.newerThan !== void 0 ? { newerThan: opts.newerThan } : {},
|
|
7328
|
+
...opts.limit !== void 0 ? { limit: opts.limit } : {}
|
|
7329
|
+
};
|
|
7330
|
+
}
|
|
7331
|
+
async function executeRun(traces, opts, color) {
|
|
7332
|
+
const shared = {
|
|
7333
|
+
...sharedRunOptions(opts),
|
|
7334
|
+
color,
|
|
7335
|
+
// Run-only, both of them: `--history` is a point in one session's
|
|
7336
|
+
// timeline, and a manifest belongs to exactly one session (ADR 01024).
|
|
7337
|
+
history: opts.history,
|
|
7338
|
+
...opts.manifest !== void 0 ? { manifest: opts.manifest } : {}
|
|
7339
|
+
};
|
|
7340
|
+
const selecting = opts.allProjects === true || opts.newerThan !== void 0 || opts.limit !== void 0;
|
|
7341
|
+
if (traces.length > 1 || selecting) {
|
|
7342
|
+
if (opts.manifest !== void 0) {
|
|
7343
|
+
throw new TracevalsError(
|
|
7344
|
+
"--manifest names one session's manifest, so it cannot be used with a corpus; drop it and each trace will find its own"
|
|
7345
|
+
);
|
|
7346
|
+
}
|
|
7347
|
+
const { runBatch } = await import("./batch-7JJWXWSF.js");
|
|
7348
|
+
const { report: report2, rendered: rendered3 } = await runBatch({
|
|
7349
|
+
...shared,
|
|
7350
|
+
...traces.length > 0 ? { traces } : {}
|
|
7351
|
+
});
|
|
7352
|
+
console.log(rendered3);
|
|
7353
|
+
process.exitCode = report2.exitCode;
|
|
7354
|
+
return;
|
|
7355
|
+
}
|
|
7356
|
+
let trace = traces[0];
|
|
7357
|
+
if (trace === void 0) {
|
|
7358
|
+
if (!process.stdout.isTTY || !process.stdin.isTTY) {
|
|
7359
|
+
throw new TracevalsError(
|
|
7360
|
+
"no trace given; pass a trace file or run `manni tracevals list`"
|
|
7361
|
+
);
|
|
7362
|
+
}
|
|
7363
|
+
const { pickTrace } = await import("./picker-6HRMETE7.js");
|
|
7364
|
+
trace = await pickTrace();
|
|
7365
|
+
}
|
|
7366
|
+
const { report, rendered: rendered2 } = await runRun2({ ...shared, tracePath: trace });
|
|
7367
|
+
console.log(rendered2);
|
|
7368
|
+
process.exitCode = report.exitCode;
|
|
7369
|
+
}
|
|
7370
|
+
function addRunFlags(cmd, options = {}) {
|
|
7371
|
+
const withHistory = options.history !== false;
|
|
7372
|
+
const formats = options.formats ?? REPORT_FORMATS3;
|
|
7373
|
+
const base = addConfigFlags(cmd).option(
|
|
7374
|
+
"--project <dir>",
|
|
7375
|
+
"artifact-lookup root (overrides the trace's recorded cwd)"
|
|
7376
|
+
).option(
|
|
7377
|
+
"--provider <name>",
|
|
7378
|
+
"Judge provider: auto (default) | anthropic | openai | claude-cli | llama-cpp"
|
|
7379
|
+
).option(
|
|
7380
|
+
"--model <model>",
|
|
7381
|
+
"Model override; needs a named provider, from here or config"
|
|
7382
|
+
).option("--local", LOCAL_FLAG_HELP).option("--runs <n>", "ensemble runs per eval", (v) => Number(v)).option("--deterministic-only", "skip LLM-judged evals").option(
|
|
7383
|
+
"--offline",
|
|
7384
|
+
"never fetch a remote manifest; read external metadata from local manifests"
|
|
7385
|
+
).option("--no-cache", "bypass the judge cache").option(
|
|
7386
|
+
"--max-turns <n>",
|
|
7387
|
+
"Stop after this many ensemble runs (a cached ensemble costs none)",
|
|
7388
|
+
(v) => Number(v)
|
|
7389
|
+
).option(
|
|
7390
|
+
"-f, --format <format>",
|
|
7391
|
+
`Output format: ${formats.join(" | ")}`,
|
|
7392
|
+
parseFormatArg("--format", formats),
|
|
7393
|
+
"pretty"
|
|
7394
|
+
).option("-o, --output <file>", "also write the report to a file").option(
|
|
7395
|
+
"--report-unused-artifacts",
|
|
7396
|
+
"list every skill and agent the session was offered and never used"
|
|
7397
|
+
).option("--fail-on-review", "Exit 1 when any eval lands in the human-review zone (the default)").option("--no-fail-on-review", "Do not fail the run on evals left for human review").option(
|
|
7398
|
+
"--allow-execution <kind>",
|
|
7399
|
+
"Run only these execution grants (repeatable): frontmatter-commands",
|
|
7400
|
+
collectGrant,
|
|
7401
|
+
[]
|
|
7402
|
+
).option("--no-execution", "Clear every execution grant for this run").option(
|
|
7403
|
+
"--require <module>",
|
|
7404
|
+
"load a grader plugin; repeatable, and added to config plugins",
|
|
7405
|
+
collect,
|
|
7406
|
+
[]
|
|
7407
|
+
).option("--all-projects", "evaluate every project's traces in the session store").option(
|
|
7408
|
+
"--newer-than <duration>",
|
|
7409
|
+
"only traces modified within this window, such as 30m, 24h, 7d or 2w"
|
|
7410
|
+
).option("--limit <n>", "maximum traces to evaluate", (v) => Number(v));
|
|
7411
|
+
return withHistory ? base.option(
|
|
7412
|
+
"--history",
|
|
7413
|
+
"append to history and compare with the previous run"
|
|
7414
|
+
) : base;
|
|
7415
|
+
}
|
|
7416
|
+
addRunFlags(
|
|
7417
|
+
// No `isDefault` (proposal 0034): `run` is a verb, so a bare
|
|
7418
|
+
// `manni tracevals` prints the help and exits 2, and a trace path with no
|
|
7419
|
+
// verb is commander's unknown-command refusal rather than a silent run.
|
|
7420
|
+
program.command("run [traces...]").description(
|
|
7421
|
+
"Evaluate one or more traces against the skills and instructions they used"
|
|
7422
|
+
)
|
|
7423
|
+
).option(
|
|
7424
|
+
"--manifest <file>",
|
|
7425
|
+
"session manifest to compare artifacts against; without it one is looked for beside the trace and under the project"
|
|
7426
|
+
).action(async (traces, opts, command) => {
|
|
7427
|
+
try {
|
|
7428
|
+
await executeRun(traces, opts, colorFor(command, process.stdout.isTTY));
|
|
7429
|
+
} catch (e) {
|
|
7430
|
+
fail(e);
|
|
7431
|
+
}
|
|
7432
|
+
});
|
|
7433
|
+
addRunFlags(
|
|
7434
|
+
program.command("calibrate [traces...]").description(
|
|
7435
|
+
"Measure judged and graded verdicts against a labels file: false passes, false fails, and review volume"
|
|
7436
|
+
).option(
|
|
7437
|
+
"--labels <file>",
|
|
7438
|
+
"calibration labels sidecar (default: calibrate.labels)"
|
|
7439
|
+
).option(
|
|
7440
|
+
"--sweep",
|
|
7441
|
+
"re-score the corpus across the configured grid of zones and ensemble sizes, from cached verdicts"
|
|
7442
|
+
).option(
|
|
7443
|
+
"--max-false-pass <n>",
|
|
7444
|
+
"exit 1 above this many false passes",
|
|
7445
|
+
(v) => Number(v)
|
|
7446
|
+
).option(
|
|
7447
|
+
"--max-false-fail <n>",
|
|
7448
|
+
"exit 1 above this many false fails",
|
|
7449
|
+
(v) => Number(v)
|
|
7450
|
+
).option(
|
|
7451
|
+
"--max-review <n>",
|
|
7452
|
+
"exit 1 above this many needs-review outcomes",
|
|
7453
|
+
(v) => Number(v)
|
|
7454
|
+
),
|
|
7455
|
+
{ history: false, formats: CALIBRATE_FORMATS }
|
|
7456
|
+
).action(
|
|
7457
|
+
async (traces, opts, command) => {
|
|
7458
|
+
try {
|
|
7459
|
+
const shared = {
|
|
7460
|
+
...sharedRunOptions(opts),
|
|
7461
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
7462
|
+
};
|
|
7463
|
+
whole("--max-false-pass", opts.maxFalsePass, 0);
|
|
7464
|
+
whole("--max-false-fail", opts.maxFalseFail, 0);
|
|
7465
|
+
whole("--max-review", opts.maxReview, 0);
|
|
7466
|
+
const { runCalibrate: runCalibrate2 } = await import("./calibrate-IA2ARFMH.js");
|
|
7467
|
+
const { report, rendered: rendered2 } = await runCalibrate2({
|
|
7468
|
+
...shared,
|
|
7469
|
+
...traces.length > 0 ? { traces } : {},
|
|
7470
|
+
...opts.labels !== void 0 ? { labels: opts.labels } : {},
|
|
7471
|
+
...opts.sweep !== void 0 ? { sweep: opts.sweep } : {},
|
|
7472
|
+
...opts.maxFalsePass !== void 0 ? { maxFalsePass: opts.maxFalsePass } : {},
|
|
7473
|
+
...opts.maxFalseFail !== void 0 ? { maxFalseFail: opts.maxFalseFail } : {},
|
|
7474
|
+
...opts.maxReview !== void 0 ? { maxReview: opts.maxReview } : {}
|
|
7475
|
+
});
|
|
7476
|
+
console.log(rendered2);
|
|
7477
|
+
process.exitCode = report.exitCode;
|
|
7478
|
+
} catch (e) {
|
|
7479
|
+
fail(e);
|
|
7480
|
+
}
|
|
7481
|
+
}
|
|
7482
|
+
);
|
|
7483
|
+
addConfigFlags(
|
|
7484
|
+
program.command("fill [paths...]").description(
|
|
7485
|
+
"Propose evals for skills, agent definitions, and project rules, and write those above the confidence threshold"
|
|
7486
|
+
)
|
|
7487
|
+
).option("--project <dir>", "project root to scan (default: current directory)").option("--exclude <glob>", "Glob to exclude (repeatable)", collect, []).option("--dry-run", "report proposals without writing them").option(
|
|
7488
|
+
"--confidence <n>",
|
|
7489
|
+
"minimum confidence to write (0-1)",
|
|
7490
|
+
(v) => parseFloat(v)
|
|
7491
|
+
).option(
|
|
7492
|
+
"--max-evals <n>",
|
|
7493
|
+
"maximum evals per artifact, including existing ones",
|
|
7494
|
+
(v) => Number(v)
|
|
7495
|
+
).option(
|
|
7496
|
+
"--max-turns <n>",
|
|
7497
|
+
"Stop after this many inference calls (a cached artifact costs none)",
|
|
7498
|
+
(v) => Number(v)
|
|
7499
|
+
).option("--no-cache", "bypass the proposal cache").option(
|
|
7500
|
+
"--provider <name>",
|
|
7501
|
+
"Provider: auto (default) | anthropic | openai | claude-cli | llama-cpp"
|
|
7502
|
+
).option(
|
|
7503
|
+
"--model <model>",
|
|
7504
|
+
"Model override; needs a named provider, from here or config"
|
|
7505
|
+
).option("--local", LOCAL_FLAG_HELP).option(
|
|
7506
|
+
"--offline",
|
|
7507
|
+
"never fetch a remote manifest; read external metadata from local manifests"
|
|
7508
|
+
).option(
|
|
7509
|
+
"--require <module>",
|
|
7510
|
+
"load a grader plugin; repeatable, and added to config plugins",
|
|
7511
|
+
collect,
|
|
7512
|
+
[]
|
|
7513
|
+
).option(
|
|
7514
|
+
"-f, --format <format>",
|
|
7515
|
+
`Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
|
|
7516
|
+
parseFormatArg("--format", SUMMARY_FORMATS2),
|
|
7517
|
+
"pretty"
|
|
7518
|
+
).action(async (paths, opts, command) => {
|
|
7519
|
+
try {
|
|
7520
|
+
numeric("--confidence", opts.confidence, 0, 1);
|
|
7521
|
+
whole("--max-evals", opts.maxEvals, 1);
|
|
7522
|
+
whole("--max-turns", opts.maxTurns, 1);
|
|
7523
|
+
const { report, rendered: rendered2 } = await runFill3({
|
|
7524
|
+
...configOptions(opts),
|
|
7525
|
+
...paths.length > 0 ? { paths } : {},
|
|
7526
|
+
...opts.project !== void 0 ? { project: opts.project } : {},
|
|
7527
|
+
...opts.exclude !== void 0 ? { exclude: opts.exclude } : {},
|
|
7528
|
+
...opts.dryRun !== void 0 ? { dryRun: opts.dryRun } : {},
|
|
7529
|
+
...opts.confidence !== void 0 ? { confidence: opts.confidence } : {},
|
|
7530
|
+
...opts.maxEvals !== void 0 ? { maxEvals: opts.maxEvals } : {},
|
|
7531
|
+
...opts.maxTurns !== void 0 ? { maxTurns: opts.maxTurns } : {},
|
|
7532
|
+
...opts.cache === false ? { noCache: true } : {},
|
|
7533
|
+
...opts.provider !== void 0 ? { provider: opts.provider } : {},
|
|
7534
|
+
...opts.model !== void 0 ? { model: opts.model } : {},
|
|
7535
|
+
...opts.local !== void 0 ? { local: opts.local } : {},
|
|
7536
|
+
...opts.offline === true ? { offline: true } : {},
|
|
7537
|
+
...opts.require !== void 0 ? { require: opts.require } : {},
|
|
7538
|
+
// P1 (proposal 0047): offered only on a terminal, where there is
|
|
7539
|
+
// someone to answer. Off one the block goes to the artifact and the
|
|
7540
|
+
// W1 line says so.
|
|
7541
|
+
confirm: terminalConfirm(),
|
|
7542
|
+
onNotice: notice,
|
|
7543
|
+
onRelocated: (result) => {
|
|
7544
|
+
const manifests = result.manifests.map((m) => m.file).join(", ");
|
|
7545
|
+
notice(`moved ${String(result.summary.moved)} value(s) into ${manifests}.`);
|
|
7546
|
+
},
|
|
7547
|
+
color: colorFor(command, process.stdout.isTTY)
|
|
7548
|
+
});
|
|
7549
|
+
console.log(
|
|
7550
|
+
opts.format === "json" ? JSON.stringify(report, null, 2) : rendered2
|
|
7551
|
+
);
|
|
7552
|
+
process.exitCode = report.exitCode;
|
|
7553
|
+
} catch (e) {
|
|
7554
|
+
fail(e);
|
|
7555
|
+
}
|
|
7556
|
+
});
|
|
7557
|
+
addConfigFlags(
|
|
7558
|
+
program.command("capture").description(
|
|
7559
|
+
"Record a session manifest: sha256 of every instruction artifact plus the git SHA, read from a Claude Code hook payload on stdin"
|
|
7560
|
+
)
|
|
7561
|
+
).option("--project <dir>", "project root to scan (default: the payload's cwd)").option("--session-id <id>", "session id (default: the payload's session_id)").option("-o, --out <file>", "write here instead of the configured directory").option(
|
|
7562
|
+
"-f, --format <format>",
|
|
7563
|
+
`Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
|
|
7564
|
+
parseFormatArg("--format", SUMMARY_FORMATS2),
|
|
7565
|
+
"pretty"
|
|
7566
|
+
).action(async (opts) => {
|
|
7567
|
+
try {
|
|
7568
|
+
const { runCapture } = await import("./capture-C32JXIFN.js");
|
|
7569
|
+
const result = await runCapture({
|
|
7570
|
+
version,
|
|
7571
|
+
...configOptions(opts),
|
|
7572
|
+
...opts.project !== void 0 ? { project: opts.project } : {},
|
|
7573
|
+
...opts.sessionId !== void 0 ? { sessionId: opts.sessionId } : {},
|
|
7574
|
+
...opts.out !== void 0 ? { out: opts.out } : {},
|
|
7575
|
+
format: opts.format ?? "pretty"
|
|
7576
|
+
});
|
|
7577
|
+
if (result.stdout !== "") console.log(result.stdout);
|
|
7578
|
+
if (result.stderr !== "") notice(result.stderr);
|
|
7579
|
+
process.exitCode = result.exitCode;
|
|
7580
|
+
} catch (e) {
|
|
7581
|
+
fail(e);
|
|
7582
|
+
}
|
|
7583
|
+
});
|
|
7584
|
+
program.command("list").description("List discoverable traces (Claude Code session files)").option(
|
|
7585
|
+
"--project <dir>",
|
|
7586
|
+
"project directory to scope to (default: current directory)"
|
|
7587
|
+
).option("--all-projects", "scan every project in the session store").option(
|
|
7588
|
+
"--newer-than <duration>",
|
|
7589
|
+
"only traces modified within this window, such as 30m, 24h, 7d or 2w"
|
|
7590
|
+
).option("--limit <n>", "maximum traces to list", (v) => Number(v)).option(
|
|
7591
|
+
"-f, --format <format>",
|
|
7592
|
+
`Output format: ${SUMMARY_FORMATS2.join(" | ")}`,
|
|
7593
|
+
parseFormatArg("--format", SUMMARY_FORMATS2),
|
|
7594
|
+
"pretty"
|
|
7595
|
+
).action(async (opts, command) => {
|
|
7596
|
+
try {
|
|
7597
|
+
whole("--limit", opts.limit, 1);
|
|
7598
|
+
const run = await runList2({
|
|
7599
|
+
project: opts.project,
|
|
7600
|
+
allProjects: opts.allProjects,
|
|
7601
|
+
...opts.newerThan !== void 0 ? { newerThan: opts.newerThan } : {},
|
|
7602
|
+
limit: opts.limit
|
|
7603
|
+
});
|
|
7604
|
+
console.log(
|
|
7605
|
+
opts.format === "json" ? JSON.stringify(run, null, 2) : renderList2(run, { color: colorFor(command, process.stdout.isTTY) })
|
|
7606
|
+
);
|
|
7607
|
+
} catch (e) {
|
|
7608
|
+
fail(e);
|
|
7609
|
+
}
|
|
7610
|
+
});
|
|
7611
|
+
return program;
|
|
7612
|
+
}
|
|
7613
|
+
|
|
7614
|
+
// src/cli.ts
|
|
7615
|
+
function buildProgram10() {
|
|
7241
7616
|
const meta = buildProgram().name("meta").description(
|
|
7242
7617
|
"Validate, read, query and fill document metadata against JSON Schema"
|
|
7243
7618
|
);
|
|
7244
7619
|
const metaCommands = new Set(meta.commands.map((c) => c.name()));
|
|
7245
|
-
const program = new
|
|
7620
|
+
const program = new Command9();
|
|
7246
7621
|
program.name("manni").description("Tools for documentation that is meant to be checked.").version(package_default.version, "-V, --version").showHelpAfterError("(add --help for usage)").exitOverride().configureOutput({
|
|
7247
7622
|
outputError: (str, write) => {
|
|
7248
7623
|
write(str);
|
|
@@ -7285,13 +7660,18 @@ function buildProgram9() {
|
|
|
7285
7660
|
"Deterministic knowledge graphs derived from documentation frontmatter and formatting"
|
|
7286
7661
|
)
|
|
7287
7662
|
);
|
|
7663
|
+
program.addCommand(
|
|
7664
|
+
buildProgram9().name("tracevals").description(
|
|
7665
|
+
"Deterministic and LLM-as-judge adherence evals for AI agent session traces"
|
|
7666
|
+
)
|
|
7667
|
+
);
|
|
7288
7668
|
program.addCommand(
|
|
7289
7669
|
buildProgram5().name("key").description("Set and rotate the family key that encrypted values are encrypted with.")
|
|
7290
7670
|
);
|
|
7291
7671
|
return program;
|
|
7292
7672
|
}
|
|
7293
|
-
runIfMain(import.meta.url,
|
|
7673
|
+
runIfMain(import.meta.url, buildProgram10);
|
|
7294
7674
|
export {
|
|
7295
|
-
|
|
7675
|
+
buildProgram10 as buildProgram
|
|
7296
7676
|
};
|
|
7297
7677
|
//# sourceMappingURL=cli.js.map
|