@hawkeyexl/manni 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/batch-7JJWXWSF.js +18 -0
- package/dist/batch-7JJWXWSF.js.map +1 -0
- package/dist/calibrate-IA2ARFMH.js +13 -0
- package/dist/calibrate-IA2ARFMH.js.map +1 -0
- package/dist/capture-C32JXIFN.js +15 -0
- package/dist/capture-C32JXIFN.js.map +1 -0
- package/dist/chunk-3R23C5DT.js +120 -0
- package/dist/chunk-3R23C5DT.js.map +1 -0
- package/dist/chunk-CI2GOU3A.js +4447 -0
- package/dist/chunk-CI2GOU3A.js.map +1 -0
- package/dist/{chunk-TH565O2N.js → chunk-DP2ZA3WH.js} +898 -278
- package/dist/chunk-DP2ZA3WH.js.map +1 -0
- package/dist/chunk-ESJ5W7EV.js +297 -0
- package/dist/chunk-ESJ5W7EV.js.map +1 -0
- package/dist/chunk-IOYXPBCL.js +37 -0
- package/dist/chunk-IOYXPBCL.js.map +1 -0
- package/dist/chunk-KTSJ2ZHA.js +40 -0
- package/dist/chunk-KTSJ2ZHA.js.map +1 -0
- package/dist/{chunk-BNHQWIAE.js → chunk-MFDD3YGU.js} +2 -2
- package/dist/chunk-MOQXWXSG.js +1941 -0
- package/dist/chunk-MOQXWXSG.js.map +1 -0
- package/dist/{chunk-UYDRDFYF.js → chunk-O6RVQKK6.js} +1943 -909
- package/dist/chunk-O6RVQKK6.js.map +1 -0
- package/dist/chunk-QROCFW5S.js +174 -0
- package/dist/chunk-QROCFW5S.js.map +1 -0
- package/dist/chunk-RVVCA4WL.js +379 -0
- package/dist/chunk-RVVCA4WL.js.map +1 -0
- package/dist/cli.js +422 -42
- package/dist/cli.js.map +1 -1
- package/dist/docmeta.js +3 -2
- package/dist/docmeta.js.map +1 -1
- package/dist/index.d.ts +3873 -795
- package/dist/index.js +292 -4
- package/dist/index.js.map +1 -1
- package/dist/picker-6HRMETE7.js +12 -0
- package/dist/picker-6HRMETE7.js.map +1 -0
- package/package.json +3 -1
- package/plugin/tracevals/.claude-plugin/plugin.json +9 -0
- package/plugin/tracevals/hooks/hooks.json +17 -0
- package/dist/chunk-TH565O2N.js.map +0 -1
- package/dist/chunk-UYDRDFYF.js.map +0 -1
- /package/dist/{chunk-BNHQWIAE.js.map → chunk-MFDD3YGU.js.map} +0 -0
package/README.md
CHANGED
|
@@ -22,7 +22,7 @@ Project](https://www.thegooddocsproject.dev/template), and the [Seven-Action
|
|
|
22
22
|
model](https://passo.uno/seven-action-model/). They also cover the front matter
|
|
23
23
|
contracts of site generators such as
|
|
24
24
|
[Docusaurus](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-content-docs#markdown-front-matter)
|
|
25
|
-
3.10, Hugo, Jekyll, and MkDocs Material. Twenty-
|
|
25
|
+
3.10, Hugo, Jekyll, and MkDocs Material. Twenty-six are manni's own metadata
|
|
26
26
|
vocabularies and their strict overlays. Run `manni meta schemas` for the full
|
|
27
27
|
list. It follows [clig.dev](https://clig.dev) conventions and returns a nonzero
|
|
28
28
|
exit code (plus optional GitHub annotations) when validation fails.
|
|
@@ -102,7 +102,7 @@ actually reads rather than hand-written:
|
|
|
102
102
|
```yaml
|
|
103
103
|
repos:
|
|
104
104
|
- repo: https://github.com/hawkeyexl/manni
|
|
105
|
-
rev: v4.
|
|
105
|
+
rev: v4.3.0
|
|
106
106
|
hooks:
|
|
107
107
|
- id: manni-meta
|
|
108
108
|
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
keepNewerThan,
|
|
4
|
+
parseNewerThan,
|
|
5
|
+
resolveBatchTraces,
|
|
6
|
+
runBatch
|
|
7
|
+
} from "./chunk-CI2GOU3A.js";
|
|
8
|
+
import "./chunk-MOQXWXSG.js";
|
|
9
|
+
import "./chunk-DP2ZA3WH.js";
|
|
10
|
+
import "./chunk-3R23C5DT.js";
|
|
11
|
+
import "./chunk-IOYXPBCL.js";
|
|
12
|
+
export {
|
|
13
|
+
keepNewerThan,
|
|
14
|
+
parseNewerThan,
|
|
15
|
+
resolveBatchTraces,
|
|
16
|
+
runBatch
|
|
17
|
+
};
|
|
18
|
+
//# sourceMappingURL=batch-7JJWXWSF.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
runCalibrate
|
|
4
|
+
} from "./chunk-RVVCA4WL.js";
|
|
5
|
+
import "./chunk-CI2GOU3A.js";
|
|
6
|
+
import "./chunk-MOQXWXSG.js";
|
|
7
|
+
import "./chunk-DP2ZA3WH.js";
|
|
8
|
+
import "./chunk-3R23C5DT.js";
|
|
9
|
+
import "./chunk-IOYXPBCL.js";
|
|
10
|
+
export {
|
|
11
|
+
runCalibrate
|
|
12
|
+
};
|
|
13
|
+
//# sourceMappingURL=calibrate-IA2ARFMH.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
renderCapture,
|
|
4
|
+
runCapture
|
|
5
|
+
} from "./chunk-ESJ5W7EV.js";
|
|
6
|
+
import "./chunk-QROCFW5S.js";
|
|
7
|
+
import "./chunk-MOQXWXSG.js";
|
|
8
|
+
import "./chunk-DP2ZA3WH.js";
|
|
9
|
+
import "./chunk-3R23C5DT.js";
|
|
10
|
+
import "./chunk-IOYXPBCL.js";
|
|
11
|
+
export {
|
|
12
|
+
renderCapture,
|
|
13
|
+
runCapture
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=capture-C32JXIFN.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
ToolError
|
|
4
|
+
} from "./chunk-IOYXPBCL.js";
|
|
5
|
+
|
|
6
|
+
// src/tracevals/trace/discover.ts
|
|
7
|
+
import { open, readdir, stat } from "node:fs/promises";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { join, resolve } from "node:path";
|
|
10
|
+
function slugFor(cwd) {
|
|
11
|
+
return cwd.replace(/[^a-zA-Z0-9]/g, "-");
|
|
12
|
+
}
|
|
13
|
+
function configDir(env = process.env) {
|
|
14
|
+
const configured = env.CLAUDE_CONFIG_DIR;
|
|
15
|
+
if (configured) return resolve(configured);
|
|
16
|
+
return join(homedir(), ".claude");
|
|
17
|
+
}
|
|
18
|
+
async function discoverTraces(options = {}) {
|
|
19
|
+
const store = join(configDir(options.env), "projects");
|
|
20
|
+
let projectDirs;
|
|
21
|
+
if (options.allProjects) {
|
|
22
|
+
try {
|
|
23
|
+
const entries = await readdir(store, { withFileTypes: true });
|
|
24
|
+
projectDirs = entries.filter((e) => e.isDirectory()).map((e) => join(store, e.name));
|
|
25
|
+
} catch {
|
|
26
|
+
return [];
|
|
27
|
+
}
|
|
28
|
+
} else {
|
|
29
|
+
const project = options.project ?? process.cwd();
|
|
30
|
+
projectDirs = [join(store, slugFor(project))];
|
|
31
|
+
}
|
|
32
|
+
const listings = [];
|
|
33
|
+
for (const dir of projectDirs) {
|
|
34
|
+
let names;
|
|
35
|
+
try {
|
|
36
|
+
names = (await readdir(dir)).filter((n) => n.endsWith(".jsonl"));
|
|
37
|
+
} catch {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
for (const name of names) {
|
|
41
|
+
const file = join(dir, name);
|
|
42
|
+
try {
|
|
43
|
+
const info = await stat(file);
|
|
44
|
+
listings.push({
|
|
45
|
+
file,
|
|
46
|
+
mtimeMs: info.mtimeMs,
|
|
47
|
+
sizeBytes: info.size,
|
|
48
|
+
...await summarizeTrace(file)
|
|
49
|
+
});
|
|
50
|
+
} catch {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
listings.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
56
|
+
return options.limit !== void 0 ? listings.slice(0, options.limit) : listings;
|
|
57
|
+
}
|
|
58
|
+
var SUMMARY_WINDOW = 256 * 1024;
|
|
59
|
+
async function readHead(file, bytes) {
|
|
60
|
+
let handle;
|
|
61
|
+
try {
|
|
62
|
+
handle = await open(file, "r");
|
|
63
|
+
const buffer = Buffer.alloc(bytes);
|
|
64
|
+
const { bytesRead } = await handle.read(buffer, 0, bytes, 0);
|
|
65
|
+
return buffer.toString("utf-8", 0, bytesRead);
|
|
66
|
+
} catch {
|
|
67
|
+
return null;
|
|
68
|
+
} finally {
|
|
69
|
+
await handle?.close();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
async function summarizeTrace(file) {
|
|
73
|
+
const content = await readHead(file, SUMMARY_WINDOW);
|
|
74
|
+
if (content === null) return {};
|
|
75
|
+
const summary = {};
|
|
76
|
+
for (const line of content.split("\n")) {
|
|
77
|
+
const trimmed = line.trim();
|
|
78
|
+
if (!trimmed) continue;
|
|
79
|
+
let parsed;
|
|
80
|
+
try {
|
|
81
|
+
parsed = JSON.parse(trimmed);
|
|
82
|
+
} catch {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
if (typeof parsed !== "object" || parsed === null) continue;
|
|
86
|
+
const rec = parsed;
|
|
87
|
+
if (summary.sessionId === void 0) {
|
|
88
|
+
const id = rec.sessionId ?? rec.session_id;
|
|
89
|
+
if (typeof id === "string") summary.sessionId = id;
|
|
90
|
+
}
|
|
91
|
+
if (summary.project === void 0 && typeof rec.cwd === "string") {
|
|
92
|
+
summary.project = rec.cwd;
|
|
93
|
+
}
|
|
94
|
+
if (summary.firstPrompt === void 0 && rec.type === "user" && rec.isSidechain !== true) {
|
|
95
|
+
const message = rec.message;
|
|
96
|
+
const content2 = message?.content;
|
|
97
|
+
if (typeof content2 === "string" && !content2.startsWith("<command-")) {
|
|
98
|
+
summary.firstPrompt = content2.split("\n")[0]?.slice(0, 200);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (summary.sessionId && summary.project && summary.firstPrompt) break;
|
|
102
|
+
}
|
|
103
|
+
return summary;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// src/tracevals/types.ts
|
|
107
|
+
var TracevalsError = class extends ToolError {
|
|
108
|
+
constructor(message) {
|
|
109
|
+
super(message);
|
|
110
|
+
this.name = "TracevalsError";
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export {
|
|
115
|
+
slugFor,
|
|
116
|
+
configDir,
|
|
117
|
+
discoverTraces,
|
|
118
|
+
TracevalsError
|
|
119
|
+
};
|
|
120
|
+
//# sourceMappingURL=chunk-3R23C5DT.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/tracevals/trace/discover.ts","../src/tracevals/types.ts"],"sourcesContent":["/**\n * Session-store discovery: find Claude Code trace files for a project (or all\n * projects) in Claude Code's own config directory.\n *\n * The location is detected, not configured: `CLAUDE_CONFIG_DIR` is Claude\n * Code's variable for that directory, so a session that has moved it is\n * followed without tracevals adding a switch of its own, and a test or CI job\n * points at a fixture tree with the same variable the agent itself reads.\n */\nimport { open, readdir, stat } from \"node:fs/promises\";\nimport { homedir } from \"node:os\";\nimport { join, resolve } from \"node:path\";\n\nexport interface TraceListing {\n /** Absolute path of the trace file. */\n file: string;\n mtimeMs: number;\n sizeBytes: number;\n sessionId?: string;\n /** The cwd recorded in the session, when readable. */\n project?: string;\n /** First non-sidechain user prompt, for identification in lists. */\n firstPrompt?: string;\n}\n\nexport interface DiscoverOptions {\n /** Project directory to scope to (its session-store slug is scanned). */\n project?: string;\n /** Scan every project directory in the store. */\n allProjects?: boolean;\n /** Maximum entries returned, applied after newest-first sorting. */\n limit?: number;\n env?: Record<string, string | undefined>;\n}\n\n/** Claude Code's project-slug rule: every non-alphanumeric character → `-`. */\nexport function slugFor(cwd: string): string {\n return cwd.replace(/[^a-zA-Z0-9]/g, \"-\");\n}\n\n/**\n * Claude Code's config directory: `CLAUDE_CONFIG_DIR` when it is set, and\n * `~/.claude` otherwise. Everything the agent keeps per user — `projects/`,\n * `skills/`, `agents/`, `commands/`, `plugins/` — hangs off it.\n */\nexport function configDir(\n env: Record<string, string | undefined> = process.env,\n): string {\n const configured = env.CLAUDE_CONFIG_DIR;\n if (configured) return resolve(configured);\n return join(homedir(), \".claude\");\n}\n\nexport async function discoverTraces(\n options: DiscoverOptions = {},\n): Promise<TraceListing[]> {\n const store = join(configDir(options.env), \"projects\");\n\n let projectDirs: string[];\n if (options.allProjects) {\n try {\n const entries = await readdir(store, { withFileTypes: true });\n projectDirs = entries\n .filter((e) => e.isDirectory())\n .map((e) => join(store, e.name));\n } catch {\n return [];\n }\n } else {\n const project = options.project ?? process.cwd();\n projectDirs = [join(store, slugFor(project))];\n }\n\n const listings: TraceListing[] = [];\n for (const dir of projectDirs) {\n let names: string[];\n try {\n names = (await readdir(dir)).filter((n) => n.endsWith(\".jsonl\"));\n } catch {\n continue;\n }\n for (const name of names) {\n const file = join(dir, name);\n try {\n const info = await stat(file);\n listings.push({\n file,\n mtimeMs: info.mtimeMs,\n sizeBytes: info.size,\n ...(await summarizeTrace(file)),\n });\n } catch {\n continue;\n }\n }\n }\n\n listings.sort((a, b) => b.mtimeMs - a.mtimeMs);\n return options.limit !== undefined\n ? listings.slice(0, options.limit)\n : listings;\n}\n\n/** Bytes read from the head of a session file when summarizing for a list. */\nconst SUMMARY_WINDOW = 256 * 1024;\n\n/** Read only the first `bytes` of a file — session files run to many MB. */\nasync function readHead(file: string, bytes: number): Promise<string | null> {\n let handle;\n try {\n handle = await open(file, \"r\");\n const buffer = Buffer.alloc(bytes);\n const { bytesRead } = await handle.read(buffer, 0, bytes, 0);\n return buffer.toString(\"utf-8\", 0, bytesRead);\n } catch {\n return null;\n } finally {\n await handle?.close();\n }\n}\n\nasync function summarizeTrace(\n file: string,\n): Promise<Pick<TraceListing, \"sessionId\" | \"project\" | \"firstPrompt\">> {\n const content = await readHead(file, SUMMARY_WINDOW);\n if (content === null) return {};\n\n const summary: Pick<TraceListing, \"sessionId\" | \"project\" | \"firstPrompt\"> =\n {};\n for (const line of content.split(\"\\n\")) {\n const trimmed = line.trim();\n if (!trimmed) continue;\n let parsed: unknown;\n try {\n parsed = JSON.parse(trimmed);\n } catch {\n continue;\n }\n // A bare `null`, number or string is valid JSON but no record. The same\n // guard `detect.ts` uses: skip the line rather than let a property read\n // throw and drop the whole file from the listing.\n if (typeof parsed !== \"object\" || parsed === null) continue;\n const rec = parsed as Record<string, unknown>;\n if (summary.sessionId === undefined) {\n const id = rec.sessionId ?? rec.session_id;\n if (typeof id === \"string\") summary.sessionId = id;\n }\n if (summary.project === undefined && typeof rec.cwd === \"string\") {\n summary.project = rec.cwd;\n }\n if (\n summary.firstPrompt === undefined &&\n rec.type === \"user\" &&\n rec.isSidechain !== true\n ) {\n const message = rec.message as Record<string, unknown> | undefined;\n const content = message?.content;\n if (typeof content === \"string\" && !content.startsWith(\"<command-\")) {\n summary.firstPrompt = content.split(\"\\n\")[0]?.slice(0, 200);\n }\n }\n if (summary.sessionId && summary.project && summary.firstPrompt) break;\n }\n return summary;\n}\n","/** Shared result model and errors for manni tracevals. */\nimport type { ConsensusResult } from \"@hawkeyexl/inference\";\nimport type {\n AvailabilityReport,\n CoverageEntry,\n} from \"./artifacts/types.js\";\nimport type { ArtifactType } from \"./artifacts/types.js\";\nimport type { ManifestReport } from \"./capture/types.js\";\nimport type { Finding } from \"./graders/types.js\";\nimport { ToolError } from \"../shared/errors.js\";\n\n/**\n * Operational error: bad usage, unreadable input, unknown format. Exits 2.\n * A `ToolError`, so the shared bin runner prints it as one line.\n */\nexport class TracevalsError extends ToolError {\n constructor(message: string) {\n super(message);\n this.name = \"TracevalsError\";\n }\n}\n\nexport type Outcome = \"pass\" | \"fail\" | \"error\" | \"needs-review\" | \"skipped\";\n\nexport interface EvalResult {\n evalName: string;\n /** Path of the artifact the eval came from. */\n artifact: string;\n artifactName: string;\n artifactType: ArtifactType;\n grader: string;\n implicit: boolean;\n /**\n * Where the eval is declared: the item of `metadata.evals` that names it, in\n * the artifact or in the manifest that supplied the block (proposal 0037).\n * `file` is relative to the working directory and forward-slashed when it is\n * inside it, absolute when it is not, and a URL for a hosted manifest.\n * `line` is 1-based, and absent when nothing records one, as for the\n * implicit eval, which no entry declares.\n */\n location: { file: string; line?: number };\n outcome: Outcome;\n /** Present for deterministic evals that produced findings. */\n findings?: Finding[];\n /** Present for judged evals. */\n consensus?: ConsensusResult;\n error?: string;\n skipReason?: string;\n /**\n * Judged evals only: uncached ensemble runs this eval spent. `0` when the\n * ensemble came from cache, or when the turn budget stopped it.\n */\n turns?: number;\n /**\n * Set when the model that judged this eval also produced what it graded.\n *\n * `session` — the judge model is the model that ran the session. This is the\n * sharpest form of the bias in either manni tool: the judge is not being\n * asked about a document it happened to draft, it is being asked whether its\n * own behavior followed the rules. `criterion` — `meta-provenance` records\n * this model proposing this assertion, so the judge wrote the question.\n *\n * Reported, never fatal: bias skews a verdict, it does not stop one forming,\n * and erroring would punish a single-model setup with no second provider to\n * reach for.\n */\n selfPreference?: { axis: \"session\" | \"criterion\"; model: string };\n /**\n * The eval's weight in the run's pass rate. Never changes its own outcome —\n * the binary verdict is what the reporters and exit code consume.\n */\n weight?: number;\n durationMs: number;\n}\n\nexport interface RunSummary {\n total: number;\n pass: number;\n fail: number;\n error: number;\n needsReview: number;\n skipped: number;\n /**\n * Weighted share of the graded set that passed, 1 when nothing was graded.\n *\n * The graded set is pass + fail + error; `needs-review` and `skipped` are in\n * neither half, so a session awaiting review neither helps nor hurts. Each\n * eval contributes its `weight` (1 unless it says otherwise), which is what\n * lets a secondary check report without dominating.\n *\n * Reported, never gated: manni tracevals has no suite targets to compare it\n * against, and inventing an exit-code rule around a number nobody configured\n * would be a gate no one asked for.\n */\n passRate: number;\n}\n\nexport interface RunReport {\n trace: {\n file: string;\n source: string;\n sessionId?: string;\n cwd: string;\n model?: string;\n turnCount: number;\n };\n /** Trace + resolution warnings, surfaced in every format. */\n warnings: string[];\n coverage: CoverageEntry[];\n /**\n * Offered versus used (ADR 01016). An observation about the session's\n * configuration; it never contributes to `summary` or `exitCode`.\n */\n availability: AvailabilityReport;\n /**\n * The session manifest this run consulted (ADR 01024), when one was found.\n * Absent means no manifest existed, which is the ordinary case — every hash\n * check then reports `skipped` and staleness stays the mtime heuristic.\n * Like `availability`, an observation: never in `summary`, never in\n * `exitCode`.\n */\n manifest?: ManifestReport;\n evalResults: EvalResult[];\n summary: RunSummary;\n /** 0 pass, 1 any fail/error (and needs-review when failOnReview). */\n exitCode: 0 | 1;\n /** Uncached ensemble runs this run spent. See `judge.maxTurns`. */\n turns: number;\n durationMs: number;\n}\n\n/* -------------------------------------------------------------------------- *\n * Batch runs (ADR 01018)\n *\n * Adherence is a rate across sessions, not a single verdict. `BatchReport`\n * sits alongside `RunReport` rather than replacing it: naming one trace still\n * yields exactly the `RunReport` above, byte for byte.\n * -------------------------------------------------------------------------- */\n\n/** Outcome tallies for one aggregate row. */\nexport interface AggregateCounts {\n pass: number;\n fail: number;\n error: number;\n needsReview: number;\n skipped: number;\n}\n\n/**\n * One artifact, or one eval within it, summed over every trace in the batch.\n *\n * Keyed by type and *name* rather than by path, so the same skill evaluated\n * across several projects is one rate — which is the fleet question this\n * report exists to answer. The paths it resolved from are kept in `artifacts`\n * so a reader can see when a row spans more than one.\n */\nexport interface AggregateRow {\n /** `<type>:<name>` for an artifact row, plus `::<evalName>` for an eval row. */\n key: string;\n artifactName: string;\n artifactType: ArtifactType;\n /** Distinct resolved paths this row was aggregated from, sorted. */\n artifacts: string[];\n /** Absent on artifact rows. */\n evalName?: string;\n /** The grader kinds seen for this row, sorted. Usually exactly one. */\n graders?: string[];\n /** Traces that produced at least one result for this row. */\n traces: number;\n /** Eval results contributing to this row. */\n total: number;\n counts: AggregateCounts;\n /** `total` minus `skipped` — the pass-rate denominator. */\n graded: number;\n /**\n * `pass / graded`, or `null` when nothing was graded. Skipped results are\n * excluded deliberately: a check that never armed is not evidence either\n * way, and folding it in would let an artifact nobody invoked report a\n * perfect score.\n */\n passRate: number | null;\n /** Traces where this row produced a `fail` or an `error`, in batch order. */\n failingTraces: string[];\n /** Traces where it produced a `needs-review`, in batch order. */\n reviewTraces: string[];\n /** Distinct skip reasons, sorted — an exhausted budget surfaces here. */\n skipReasons: string[];\n}\n\n/** One trace's place in the batch. */\nexport interface BatchTraceEntry {\n file: string;\n sessionId?: string;\n /**\n * Set when the trace could not be evaluated at all — unreadable, or not a\n * recognized format. One bad file in a corpus of fifty must not cost the\n * other forty-nine, so this is an entry rather than an aborted run.\n */\n error?: string;\n /** Absent when `error` is set. */\n summary?: RunSummary;\n warnings: string[];\n exitCode: 0 | 1;\n /** Judge turns spent on that trace. */\n turns: number;\n durationMs: number;\n}\n\nexport interface BatchSummary extends RunSummary {\n /** Traces selected, including any that errored. */\n traces: number;\n tracesPassed: number;\n /** Traces that produced a report with exit code 1. */\n tracesFailed: number;\n /** Traces that could not be evaluated at all. */\n tracesErrored: number;\n}\n\nexport interface BatchReport {\n traces: BatchTraceEntry[];\n /** Per-artifact rates, sorted by `key`. */\n artifacts: AggregateRow[];\n /** Per-eval rates, sorted by `key`. */\n evals: AggregateRow[];\n /** Eval counts summed over every trace, plus the per-trace tallies. */\n summary: BatchSummary;\n /** Batch-level warnings: plugin loading and trace selection. */\n warnings: string[];\n /** `1` when any trace failed or errored. */\n exitCode: 0 | 1;\n /** Judge turns spent across the batch. The budget spans it, not each trace. */\n turns: number;\n durationMs: number;\n}\n"],"mappings":";;;;;;AASA,SAAS,MAAM,SAAS,YAAY;AACpC,SAAS,eAAe;AACxB,SAAS,MAAM,eAAe;AAyBvB,SAAS,QAAQ,KAAqB;AAC3C,SAAO,IAAI,QAAQ,iBAAiB,GAAG;AACzC;AAOO,SAAS,UACd,MAA0C,QAAQ,KAC1C;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,WAAY,QAAO,QAAQ,UAAU;AACzC,SAAO,KAAK,QAAQ,GAAG,SAAS;AAClC;AAEA,eAAsB,eACpB,UAA2B,CAAC,GACH;AACzB,QAAM,QAAQ,KAAK,UAAU,QAAQ,GAAG,GAAG,UAAU;AAErD,MAAI;AACJ,MAAI,QAAQ,aAAa;AACvB,QAAI;AACF,YAAM,UAAU,MAAM,QAAQ,OAAO,EAAE,eAAe,KAAK,CAAC;AAC5D,oBAAc,QACX,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,EAC7B,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,IAAI,CAAC;AAAA,IACnC,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF,OAAO;AACL,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,kBAAc,CAAC,KAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAAA,EAC9C;AAEA,QAAM,WAA2B,CAAC;AAClC,aAAW,OAAO,aAAa;AAC7B,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,SAAS,QAAQ,CAAC;AAAA,IACjE,QAAQ;AACN;AAAA,IACF;AACA,eAAW,QAAQ,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,IAAI;AAC3B,UAAI;AACF,cAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,iBAAS,KAAK;AAAA,UACZ;AAAA,UACA,SAAS,KAAK;AAAA,UACd,WAAW,KAAK;AAAA,UAChB,GAAI,MAAM,eAAe,IAAI;AAAA,QAC/B,CAAC;AAAA,MACH,QAAQ;AACN;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,WAAS,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO;AAC7C,SAAO,QAAQ,UAAU,SACrB,SAAS,MAAM,GAAG,QAAQ,KAAK,IAC/B;AACN;AAGA,IAAM,iBAAiB,MAAM;AAG7B,eAAe,SAAS,MAAc,OAAuC;AAC3E,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,KAAK,MAAM,GAAG;AAC7B,UAAM,SAAS,OAAO,MAAM,KAAK;AACjC,UAAM,EAAE,UAAU,IAAI,MAAM,OAAO,KAAK,QAAQ,GAAG,OAAO,CAAC;AAC3D,WAAO,OAAO,SAAS,SAAS,GAAG,SAAS;AAAA,EAC9C,QAAQ;AACN,WAAO;AAAA,EACT,UAAE;AACA,UAAM,QAAQ,MAAM;AAAA,EACtB;AACF;AAEA,eAAe,eACb,MACsE;AACtE,QAAM,UAAU,MAAM,SAAS,MAAM,cAAc;AACnD,MAAI,YAAY,KAAM,QAAO,CAAC;AAE9B,QAAM,UACJ,CAAC;AACH,aAAW,QAAQ,QAAQ,MAAM,IAAI,GAAG;AACtC,UAAM,UAAU,KAAK,KAAK;AAC1B,QAAI,CAAC,QAAS;AACd,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,OAAO;AAAA,IAC7B,QAAQ;AACN;AAAA,IACF;AAIA,QAAI,OAAO,WAAW,YAAY,WAAW,KAAM;AACnD,UAAM,MAAM;AACZ,QAAI,QAAQ,cAAc,QAAW;AACnC,YAAM,KAAK,IAAI,aAAa,IAAI;AAChC,UAAI,OAAO,OAAO,SAAU,SAAQ,YAAY;AAAA,IAClD;AACA,QAAI,QAAQ,YAAY,UAAa,OAAO,IAAI,QAAQ,UAAU;AAChE,cAAQ,UAAU,IAAI;AAAA,IACxB;AACA,QACE,QAAQ,gBAAgB,UACxB,IAAI,SAAS,UACb,IAAI,gBAAgB,MACpB;AACA,YAAM,UAAU,IAAI;AACpB,YAAMA,WAAU,SAAS;AACzB,UAAI,OAAOA,aAAY,YAAY,CAACA,SAAQ,WAAW,WAAW,GAAG;AACnE,gBAAQ,cAAcA,SAAQ,MAAM,IAAI,EAAE,CAAC,GAAG,MAAM,GAAG,GAAG;AAAA,MAC5D;AAAA,IACF;AACA,QAAI,QAAQ,aAAa,QAAQ,WAAW,QAAQ,YAAa;AAAA,EACnE;AACA,SAAO;AACT;;;ACrJO,IAAM,iBAAN,cAA6B,UAAU;AAAA,EAC5C,YAAY,SAAiB;AAC3B,UAAM,OAAO;AACb,SAAK,OAAO;AAAA,EACd;AACF;","names":["content"]}
|