@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.25

package/README.md

@@ -42,8 +42,8 @@ A case passes when its deterministic checks hold (`must_contain` / `must_not_con
 `should_contain_any`) and, if a `judge` rubric is set, the LLM judge rates the answer at or above
 `pass_threshold` (0–10 scale, suite default `6`). `gth eval` prints a `PASS`/`FAIL` line per case
 plus a suite total, writes structured per-case JSON and a `results.json` summary to a timestamped
-output dir (override with `-o <dir>`), and exits non-zero if any case failed. Add `-j <n>` to cap
-in-flight cases.
+output dir (override with `-o <dir>`), and exits non-zero if any case failed. `-j <n>` caps
+in-flight cases (default `1` — cases run one at a time until you ask for more).
 
 ### Examples
 
@@ -59,8 +59,9 @@ gth workflow rank-models.mjs --args '{"topic":"robotics"}'
 comma-separated; omit to use the configured model) and/or content-bound input rows (`--over
 <file.csv|file.jsonl>` — one cell per row, with `{{field}}` placeholders bound from the row). It
 writes the same structured per-cell output as `eval` but — unlike `eval` — exits `0` as long as the
-cells *ran*: a poor answer is not a harness failure. `-j <n>` caps concurrency, `--retry <n>` retries
-a failed cell (default `0`), `-o <dir>` sets the output dir.
+cells *ran*: a poor answer is not a harness failure. `-j <n>` caps concurrency (default `1` — cells
+run one at a time until you ask for more), `--retry <n>` retries a failed cell (default `0`), `-o
+<dir>` sets the output dir.
 
 `gth workflow <script.mjs>` runs a local ESM script whose default export is `async (ctx) => result`;
 the return value is printed (a string as-is, anything else as pretty JSON), and `--args <json>` is
@@ -108,10 +109,11 @@ import { runEvalSuite } from '@gaunt-sloth/batch/evalRunner.js';
 The public API (see the package's `index.ts`) groups by command:
 
 - **Matrix** (`gth batch`): `buildMatrix`, `bindCellContent`, `parseOverFile`, `runBatchMatrix`,
-  `buildBatchSummary`, `writeBatchOutput`, `DEFAULT_CONCURRENCY`.
+  `buildBatchSummary`, `writeBatchOutput`, `DEFAULT_CELL_CONCURRENCY`.
 - **Eval** (`gth eval`): `parseEvalSuite`, `runDeterministicChecks`, `judgeEvalCase`, `runEvalSuite`,
   `writeEvalOutput`, `EvalVerdictSchema`, `DEFAULT_EVAL_PASS_THRESHOLD`.
-- **Workflow** (`gth workflow`): `runWorkflow`.
+- **Workflow** (`gth workflow`): `runWorkflow`, `DEFAULT_WORKFLOW_CONCURRENCY` (the
+  `ctx.parallel()` fan-out cap, independent of the cell default).
 
 The corresponding TypeScript types (`MatrixCell`, `BatchSummary`, `CellResult`, `EvalSuite`,
 `EvalCaseResult`, `WorkflowContext`, …) are exported alongside them.

package/dist/BatchRunner.d.ts

@@ -14,5 +14,25 @@ import { type BatchRunnerOptions, type BatchSummary, type CellResult, type Matri
  * up with the matrix.
  */
 export declare function runBatchMatrix(cells: MatrixCell[], options: BatchRunnerOptions): Promise<CellResult[]>;
+/**
+ * BATCH-24 — build the end-of-run nudge that makes the serial default discoverable (DL-1: the
+ * user should never have to guess why a big matrix took its time). Returns `undefined` when there
+ * is nothing worth saying: the user already chose a `-j`, or only one unit ran (nothing to
+ * parallelize).
+ *
+ * Deliberately names **no number**. The right `-j` depends entirely on the backend — a local
+ * single-GPU ollama wants exactly this serial default, a cloud key with headroom can take many —
+ * so printing "try -j 4" would hand the wrong advice to the very setup this default protects.
+ *
+ * Kept out of the package root's export list because it is a detail two first-party commands
+ * share rather than something the plugin API advertises. Note that is discoverability only, not
+ * a boundary: the package publishes a `./*.js` subpath map, so `@gaunt-sloth/batch/BatchRunner.js`
+ * — the specifier both commands import it by — is reachable by any consumer.
+ *
+ * @param unitCount - how many cells/cases the run actually processed
+ * @param explicitConcurrency - the user's `-j/--concurrency` value, `undefined` when not supplied
+ * @param noun - what the units are called on this surface (`batch` cells vs `eval` cases)
+ */
+export declare function concurrencyHint(unitCount: number, explicitConcurrency: number | undefined, noun?: 'Cells' | 'Cases'): string | undefined;
 /** Build the lightweight aggregate "flake report" (pass/fail counts) from the per-cell results. */
 export declare function buildBatchSummary(results: CellResult[]): BatchSummary;

package/dist/BatchRunner.js

@@ -1,4 +1,4 @@
-import { DEFAULT_CONCURRENCY, } from '#src/types.js';
+import { DEFAULT_CELL_CONCURRENCY, } from '#src/types.js';
 /**
  * Run every cell of the matrix through the injected {@link RunCellFn}, capping in-flight work at
  * `concurrency` and retrying a failed cell up to `retry` times.
@@ -70,10 +70,36 @@ async function runCellWithRetry(cell, runCell, retry) {
 }
 function normalizeConcurrency(concurrency) {
     if (concurrency === undefined || !Number.isFinite(concurrency) || concurrency < 1) {
-        return DEFAULT_CONCURRENCY;
+        return DEFAULT_CELL_CONCURRENCY;
     }
     return Math.floor(concurrency);
 }
+/**
+ * BATCH-24 — build the end-of-run nudge that makes the serial default discoverable (DL-1: the
+ * user should never have to guess why a big matrix took its time). Returns `undefined` when there
+ * is nothing worth saying: the user already chose a `-j`, or only one unit ran (nothing to
+ * parallelize).
+ *
+ * Deliberately names **no number**. The right `-j` depends entirely on the backend — a local
+ * single-GPU ollama wants exactly this serial default, a cloud key with headroom can take many —
+ * so printing "try -j 4" would hand the wrong advice to the very setup this default protects.
+ *
+ * Kept out of the package root's export list because it is a detail two first-party commands
+ * share rather than something the plugin API advertises. Note that is discoverability only, not
+ * a boundary: the package publishes a `./*.js` subpath map, so `@gaunt-sloth/batch/BatchRunner.js`
+ * — the specifier both commands import it by — is reachable by any consumer.
+ *
+ * @param unitCount - how many cells/cases the run actually processed
+ * @param explicitConcurrency - the user's `-j/--concurrency` value, `undefined` when not supplied
+ * @param noun - what the units are called on this surface (`batch` cells vs `eval` cases)
+ */
+export function concurrencyHint(unitCount, explicitConcurrency, noun = 'Cells') {
+    if (explicitConcurrency !== undefined)
+        return undefined;
+    if (unitCount <= 1)
+        return undefined;
+    return `${noun} ran one at a time. Pass -j <n> to run them in parallel.`;
+}
 function normalizeRetry(retry) {
     if (retry === undefined || !Number.isFinite(retry) || retry < 0) {
         return 0;

package/dist/BatchRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"BatchRunner.js","sourceRoot":"","sources":["../src/BatchRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,GAMpB,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAmB,EACnB,OAA2B;IAE3B,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAEtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEvE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAgB,EAChB,OAAkB,EAClB,KAAa;IAEb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,uEAAuE;IACvE,SAAS,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,EAAE,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACnC,OAAO;oBACL,GAAG,OAAO;oBACV,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,8DAA8D;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACrB,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,qEAAqE;QACvE,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,WAA+B;IAC3D,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAClF,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,OAAqB;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,MAAM;QACN,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM;QAC7B,KAAK;KACN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"BatchRunner.js","sourceRoot":"","sources":["../src/BatchRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,GAMzB,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAmB,EACnB,OAA2B;IAE3B,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAEtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEvE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAgB,EAChB,OAAkB,EAClB,KAAa;IAEb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,uEAAuE;IACvE,SAAS,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,EAAE,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACnC,OAAO;oBACL,GAAG,OAAO;oBACV,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,8DAA8D;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACrB,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,qEAAqE;QACvE,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,WAA+B;IAC3D,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAClF,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAiB,EACjB,mBAAuC,EACvC,IAAI,GAAsB,OAAO;IAEjC,IAAI,mBAAmB,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxD,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrC,OAAO,GAAG,IAAI,0DAA0D,CAAC;AAC3E,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,OAAqB;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,MAAM;QACN,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM;QAC7B,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/blindExport.d.ts

@@ -0,0 +1,88 @@
+import type { EvalSuite } from '#src/evalTypes.js';
+/**
+ * BATCH-25 — the blind export and its round trip.
+ *
+ * A corpus labelled by one person is a corpus with one person's blind spots in it. The approvals
+ * corpus plan requires a **blind relabel by a second person** before any number from it gates a
+ * decision, and this module is what makes that structurally possible rather than aspirational: the
+ * export carries the case id, the input, and the family tags, and NOTHING else — no expected label,
+ * no expected action, no rationale, no judge rubric, no per-identity expectation blocks.
+ *
+ * The diff then compares the second labeller's file against the corpus BY ID, and — this is the
+ * part that is easy to get wrong — reports ids present in only one of the two. A relabel that
+ * silently omitted ten cases would otherwise read as 100% agreement on the sixty-eight it did
+ * cover, which is the same "a number that looks better than it earned" failure the metric layer
+ * exists to prevent.
+ */
+/** One exported case. `inputs` carries every round in order, so a multi-round negotiation case is
+ * relabellable rather than silently dropped — dropping it would be a silent cap on the export. */
+export interface BlindExportCase {
+    id: string;
+    inputs: string[];
+    tags: string[];
+}
+/** The blind export document. `note` is written into the file itself so a labeller opening it
+ * out of context knows what is being asked and that the labels were deliberately withheld. */
+export interface BlindExport {
+    note: string;
+    /** The suite's declared label enum — the labeller needs the vocabulary; that is not a leak of the
+     * answers. Empty when the suite declares no `classification:` block. */
+    labels: string[];
+    /** The suite's declared action enum, when it has one. */
+    actions: string[];
+    cases: BlindExportCase[];
+}
+/**
+ * Build the blind export for a suite.
+ *
+ * Note what is NOT here: `expectLabel`, `expectAction`, `judgeRubric`, every content assertion, and
+ * the pass threshold. An expectation block leaks the answer as surely as the label does — a
+ * `must_contain: ["refused"]` tells the labeller what was expected — so the export is built from an
+ * allow-list of three fields rather than by deleting fields from the case.
+ */
+export declare function buildBlindExport(suite: EvalSuite): BlindExport;
+/** One case as returned by the second labeller. */
+export interface RelabelEntry {
+    id: string;
+    label?: string;
+    action?: string;
+    note?: string;
+}
+/** One case where the two labellings differ. */
+export interface RelabelDisagreement {
+    id: string;
+    tags: string[];
+    corpusLabel?: string;
+    relabelLabel?: string;
+    corpusAction?: string;
+    relabelAction?: string;
+    note?: string;
+}
+/** The result of a blind relabel diff. */
+export interface RelabelDiff {
+    /** Cases present in BOTH files — the only ones agreement can be computed over. */
+    compared: number;
+    agreed: number;
+    disagreements: RelabelDisagreement[];
+    /** Ids in the corpus that the relabel file never mentions. Reported explicitly: without this a
+     * partial relabel reads as full agreement on the part it covered. */
+    missingFromRelabel: string[];
+    /** Ids in the relabel file that the corpus does not contain (a typo, or a stale export). */
+    unknownInRelabel: string[];
+    /** Everything that bounds what the agreement figure covers. Empty is the good case. */
+    warnings: string[];
+}
+/**
+ * Diff a second labeller's file against the corpus, by id.
+ *
+ * Agreement is computed ONLY over ids present in both files, and the ids present in only one are
+ * reported separately and warned about. A relabel that covers half the corpus produces an
+ * agreement figure over half the corpus and says so — it never produces a figure that looks like it
+ * covers all of it.
+ *
+ * A relabel entry with no `label` is treated as "not relabelled": it counts as missing rather than
+ * as a disagreement with `undefined`, because an unanswered case is not a dissent.
+ */
+export declare function diffRelabel(suite: EvalSuite, entries: RelabelEntry[]): RelabelDiff;
+/** Render a {@link RelabelDiff} as plain lines for the console. */
+export declare function renderRelabelDiff(diff: RelabelDiff): string[];

package/dist/blindExport.js

@@ -0,0 +1,129 @@
+const BLIND_NOTE = 'Blind relabel export. Each case carries only its id, its input(s) and its family tags — the ' +
+    'authored expected label/action and every rationale are deliberately withheld. Add a `label` ' +
+    '(and, where you have a view, an `action`) to each case and return the file; `gth eval ' +
+    '--relabel-diff` compares it to the corpus by id.';
+/**
+ * Build the blind export for a suite.
+ *
+ * Note what is NOT here: `expectLabel`, `expectAction`, `judgeRubric`, every content assertion, and
+ * the pass threshold. An expectation block leaks the answer as surely as the label does — a
+ * `must_contain: ["refused"]` tells the labeller what was expected — so the export is built from an
+ * allow-list of three fields rather than by deleting fields from the case.
+ */
+export function buildBlindExport(suite) {
+    return {
+        note: BLIND_NOTE,
+        labels: suite.classification?.labels ?? [],
+        actions: suite.classification?.actions ?? [],
+        cases: suite.cases.map((evalCase) => ({
+            id: evalCase.id,
+            inputs: evalCase.turns.map((turn) => turn.user),
+            tags: evalCase.tags ?? [],
+        })),
+    };
+}
+/**
+ * Diff a second labeller's file against the corpus, by id.
+ *
+ * Agreement is computed ONLY over ids present in both files, and the ids present in only one are
+ * reported separately and warned about. A relabel that covers half the corpus produces an
+ * agreement figure over half the corpus and says so — it never produces a figure that looks like it
+ * covers all of it.
+ *
+ * A relabel entry with no `label` is treated as "not relabelled": it counts as missing rather than
+ * as a disagreement with `undefined`, because an unanswered case is not a dissent.
+ */
+export function diffRelabel(suite, entries) {
+    // The corpus's own view of each case: the first expected label/action any expectation declares.
+    const corpus = new Map();
+    for (const evalCase of suite.cases) {
+        const blocks = evalCase.turns.flatMap((turn) => turn.expectations);
+        corpus.set(evalCase.id, {
+            tags: evalCase.tags ?? [],
+            label: blocks.find((block) => block.expectLabel !== undefined)?.expectLabel,
+            action: blocks.find((block) => block.expectAction !== undefined)?.expectAction,
+        });
+    }
+    const relabelled = new Map();
+    const unknownInRelabel = [];
+    const duplicateIds = [];
+    for (const entry of entries) {
+        if (!corpus.has(entry.id)) {
+            unknownInRelabel.push(entry.id);
+            continue;
+        }
+        if (relabelled.has(entry.id))
+            duplicateIds.push(entry.id);
+        // A blank label means "not relabelled" — it belongs in `missingFromRelabel`, not in the
+        // agreement denominator.
+        if (entry.label === undefined || entry.label.trim().length === 0)
+            continue;
+        relabelled.set(entry.id, entry);
+    }
+    const disagreements = [];
+    const missingFromRelabel = [];
+    let compared = 0;
+    let agreed = 0;
+    for (const [id, corpusCase] of corpus) {
+        const entry = relabelled.get(id);
+        if (!entry) {
+            missingFromRelabel.push(id);
+            continue;
+        }
+        compared += 1;
+        const labelAgrees = corpusCase.label === entry.label;
+        // An action is only compared when the relabeller expressed one — silence is not dissent.
+        const actionAgrees = entry.action === undefined || entry.action.trim().length === 0
+            ? true
+            : corpusCase.action === entry.action;
+        if (labelAgrees && actionAgrees) {
+            agreed += 1;
+            continue;
+        }
+        disagreements.push({
+            id,
+            tags: corpusCase.tags,
+            corpusLabel: corpusCase.label,
+            relabelLabel: entry.label,
+            corpusAction: corpusCase.action,
+            relabelAction: entry.action,
+            note: entry.note,
+        });
+    }
+    const warnings = [];
+    if (missingFromRelabel.length > 0) {
+        warnings.push(`${missingFromRelabel.length}/${corpus.size} corpus case(s) were NOT relabelled, so the ` +
+            `agreement figure covers ${compared}/${corpus.size} cases, not the whole corpus ` +
+            `(missing: ${missingFromRelabel.join(', ')}).`);
+    }
+    if (unknownInRelabel.length > 0) {
+        warnings.push(`${unknownInRelabel.length} relabel entr(ies) name ids the corpus does not contain — a typo, ` +
+            `or a stale export (${unknownInRelabel.join(', ')}).`);
+    }
+    if (duplicateIds.length > 0) {
+        warnings.push(`${duplicateIds.length} id(s) appear more than once in the relabel file; the LAST entry won ` +
+            `(${[...new Set(duplicateIds)].join(', ')}).`);
+    }
+    return { compared, agreed, disagreements, missingFromRelabel, unknownInRelabel, warnings };
+}
+/** Render a {@link RelabelDiff} as plain lines for the console. */
+export function renderRelabelDiff(diff) {
+    const lines = ['', 'BLIND RELABEL DIFF'];
+    const percent = diff.compared === 0 ? 'n/a (0 cases)' : `${((diff.agreed / diff.compared) * 100).toFixed(1)}%`;
+    lines.push(`  agreement: ${diff.agreed}/${diff.compared} compared case(s) (${percent})`);
+    for (const warning of diff.warnings)
+        lines.push(`  ! ${warning}`);
+    if (diff.disagreements.length > 0) {
+        lines.push('');
+        lines.push(`  disagreements (${diff.disagreements.length}):`);
+        for (const item of diff.disagreements) {
+            const tags = item.tags.length > 0 ? ` [${item.tags.join(', ')}]` : '';
+            lines.push(`    ${item.id}${tags}: corpus=${item.corpusLabel ?? '(none)'}` +
+                `/${item.corpusAction ?? '(none)'} vs relabel=${item.relabelLabel ?? '(none)'}` +
+                `/${item.relabelAction ?? '(none)'}` +
+                (item.note ? ` — ${item.note}` : ''));
+        }
+    }
+    return lines;
+}
+//# sourceMappingURL=blindExport.js.map

package/dist/blindExport.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"blindExport.js","sourceRoot":"","sources":["../src/blindExport.ts"],"names":[],"mappings":"AAsCA,MAAM,UAAU,GACd,8FAA8F;IAC9F,8FAA8F;IAC9F,wFAAwF;IACxF,kDAAkD,CAAC;AAErD;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAgB;IAC/C,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,MAAM,EAAE,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,EAAE;QAC1C,OAAO,EAAE,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE;QAC5C,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACpC,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/C,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;SAC1B,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAoCD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB,EAAE,OAAuB;IACnE,gGAAgG;IAChG,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+D,CAAC;IACtF,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE;YACtB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;YACzB,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,WAAW;YAC3E,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,EAAE,YAAY;SAC/E,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;YAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QACD,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1D,wFAAwF;QACxF,yBAAyB;QACzB,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC3E,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,aAAa,GAA0B,EAAE,CAAC;IAChD,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QACD,QAAQ,IAAI,CAAC,CAAC;QACd,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC;QACrD,yFAAyF;QACzF,MAAM,YAAY,GAChB,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAC5D,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC;QACzC,IAAI,WAAW,IAAI,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,CAAC;YACZ,SAAS;QACX,CAAC;QACD,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE;YACF,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,WAAW,EAAE,UAAU,CAAC,KAAK;YAC7B,YAAY,EAAE,KAAK,CAAC,KAAK;YACzB,YAAY,EAAE,UAAU,CAAC,MAAM;YAC/B,aAAa,EAAE,KAAK,CAAC,MAAM;YAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CACX,GAAG,kBAAkB,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,8CAA8C;YACvF,2BAA2B,QAAQ,IAAI,MAAM,CAAC,IAAI,+BAA+B;YACjF,aAAa,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CACjD,CAAC;IACJ,CAAC;IACD,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,CACX,GAAG,gBAAgB,CAAC,MAAM,oEAAoE;YAC5F,sBAAsB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CACxD,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CACX,GAAG,YAAY,CAAC,MAAM,uEAAuE;YAC3F,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAChD,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;AAC7F,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,iBAAiB,CAAC,IAAiB;IACjD,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAC;IACnD,MAAM,OAAO,GACX,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,sBAAsB,OAAO,GAAG,CAAC,CAAC;IACzF,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,KAAK,CAAC,IAAI,CACR,OAAO,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE;gBAC7D,IAAI,IAAI,CAAC,YAAY,IAAI,QAAQ,eAAe,IAAI,CAAC,YAAY,IAAI,QAAQ,EAAE;gBAC/E,IAAI,IAAI,CAAC,aAAa,IAAI,QAAQ,EAAE;gBACpC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACvC,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/classification.d.ts

@@ -0,0 +1,52 @@
+import type { ClassificationExtractor, ClassifiedCell, EvalClassificationSpec, EvalConfusionMatrix } from '#src/classificationTypes.js';
+/**
+ * BATCH-25 — read a classification value out of one cell's answer, and build confusion matrices
+ * from the graded results.
+ *
+ * Both halves obey the same rule: **nothing is ever dropped.** An answer that matches no declared
+ * value becomes {@link UNRECOGNIZED_LABEL}; a cell that never ran is counted as `excluded` and said
+ * out loud. A classifier report that quietly discards the rows it could not interpret reports a
+ * cleaner number than it earned, which is the failure mode this whole cluster exists to prevent.
+ */
+/**
+ * Extract one declared enum value from an answer.
+ *
+ * TOTAL by construction — the return is always either a member of `allowed` or
+ * {@link UNRECOGNIZED_LABEL}. There is deliberately no fuzzy/substring fallback: matching `safe`
+ * inside "this is not safe" is exactly the silent misreading a classifier eval must not do.
+ *
+ * Matching is case-insensitive and whitespace-trimmed (a model that answers `"Safe\n"` meant
+ * `safe`), and surrounding quotes/backticks/full stops are stripped, because those are formatting,
+ * not content. Anything beyond that is the author's job via `json_path`.
+ *
+ * @param answer The cell's raw answer text (or the resolved JSON-path value, stringified).
+ * @param extractor How to read it.
+ * @param allowed The suite's declared enum for this dimension.
+ * @returns The matched enum value, or {@link UNRECOGNIZED_LABEL}.
+ */
+export declare function extractClassificationValue(answer: string | undefined, extractor: ClassificationExtractor, allowed: string[]): string;
+/** The text an extractor actually reads — kept separate from matching so the per-cell record can
+ * carry it for diagnosis (`(unrecognized)` with no trace of what was said is not diagnosable).
+ * `undefined` = the extractor found nothing at all (no answer, unparseable JSON, unresolved path). */
+export declare function readRaw(answer: string | undefined, extractor: ClassificationExtractor): string | undefined;
+/**
+ * Build one confusion matrix (rows = expected, columns = actual) over the given cells.
+ *
+ * Axis construction is deliberate: rows are the declared enum in DECLARED order (so two runs of the
+ * same suite are diffable line by line) plus {@link NO_EXPECTATION} only when some cell declares no
+ * expectation; columns are the declared enum plus {@link UNRECOGNIZED_LABEL} and/or
+ * {@link NO_EXPECTATION} only when some cell actually produced one. Extra buckets never appear
+ * speculatively, and never fail to appear when they are populated.
+ *
+ * `counted + excluded === cells.length`, always — asserted in the unit suite, because a matrix that
+ * silently drops the cells it could not place reads as "covered everything" when it didn't.
+ */
+export declare function buildConfusionMatrix(cells: ClassifiedCell[], dimension: 'label' | 'action', declared: string[]): EvalConfusionMatrix;
+/** Every tag any cell declares, sorted — the per-tag axis for matrices and metrics. */
+export declare function collectTags(cells: ClassifiedCell[]): string[];
+/**
+ * Which enum a `classification` spec declares for a dimension. `actions` is `[]` when the suite has
+ * no action dimension, which is what makes an `expect_action` assertion a parse error rather than a
+ * silently ungradeable one.
+ */
+export declare function declaredValues(spec: EvalClassificationSpec, dimension: 'label' | 'action'): string[];

package/dist/classification.js

@@ -0,0 +1,140 @@
+import { resolveJsonPath } from '#src/deterministicChecks.js';
+import { NO_EXPECTATION, UNRECOGNIZED_LABEL } from '#src/classificationTypes.js';
+/**
+ * BATCH-25 — read a classification value out of one cell's answer, and build confusion matrices
+ * from the graded results.
+ *
+ * Both halves obey the same rule: **nothing is ever dropped.** An answer that matches no declared
+ * value becomes {@link UNRECOGNIZED_LABEL}; a cell that never ran is counted as `excluded` and said
+ * out loud. A classifier report that quietly discards the rows it could not interpret reports a
+ * cleaner number than it earned, which is the failure mode this whole cluster exists to prevent.
+ */
+/**
+ * Extract one declared enum value from an answer.
+ *
+ * TOTAL by construction — the return is always either a member of `allowed` or
+ * {@link UNRECOGNIZED_LABEL}. There is deliberately no fuzzy/substring fallback: matching `safe`
+ * inside "this is not safe" is exactly the silent misreading a classifier eval must not do.
+ *
+ * Matching is case-insensitive and whitespace-trimmed (a model that answers `"Safe\n"` meant
+ * `safe`), and surrounding quotes/backticks/full stops are stripped, because those are formatting,
+ * not content. Anything beyond that is the author's job via `json_path`.
+ *
+ * @param answer The cell's raw answer text (or the resolved JSON-path value, stringified).
+ * @param extractor How to read it.
+ * @param allowed The suite's declared enum for this dimension.
+ * @returns The matched enum value, or {@link UNRECOGNIZED_LABEL}.
+ */
+export function extractClassificationValue(answer, extractor, allowed) {
+    const raw = readRaw(answer, extractor);
+    if (raw === undefined)
+        return UNRECOGNIZED_LABEL;
+    const normalized = normalizeValue(raw);
+    if (normalized.length === 0)
+        return UNRECOGNIZED_LABEL;
+    const hit = allowed.find((candidate) => candidate.toLowerCase() === normalized);
+    return hit ?? UNRECOGNIZED_LABEL;
+}
+/** The text an extractor actually reads — kept separate from matching so the per-cell record can
+ * carry it for diagnosis (`(unrecognized)` with no trace of what was said is not diagnosable).
+ * `undefined` = the extractor found nothing at all (no answer, unparseable JSON, unresolved path). */
+export function readRaw(answer, extractor) {
+    if (answer === undefined)
+        return undefined;
+    if (extractor.kind === 'answer')
+        return answer;
+    // json_path: parse the answer as JSON and resolve the path with the SAME minimal resolver the
+    // `json_path` assertion uses, so an author who knows one knows the other. A non-JSON answer or an
+    // unresolved path yields `undefined` → `(unrecognized)`, never a throw mid-run.
+    let root;
+    try {
+        root = JSON.parse(answer);
+    }
+    catch {
+        return undefined;
+    }
+    const { found, value } = resolveJsonPath(root, extractor.path);
+    if (!found || value === undefined || value === null)
+        return undefined;
+    return typeof value === 'string' ? value : JSON.stringify(value);
+}
+/** Trim, strip wrapping quotes/backticks and a trailing full stop, lowercase. Formatting only —
+ * never content. */
+function normalizeValue(raw) {
+    let text = raw.trim();
+    // Strip one layer of wrapping quotes/backticks, then a single trailing full stop.
+    const quotes = ['"', "'", '`'];
+    while (text.length >= 2 && quotes.includes(text[0]) && text[text.length - 1] === text[0]) {
+        text = text.slice(1, -1).trim();
+    }
+    if (text.endsWith('.'))
+        text = text.slice(0, -1).trim();
+    return text.toLowerCase();
+}
+/**
+ * Build one confusion matrix (rows = expected, columns = actual) over the given cells.
+ *
+ * Axis construction is deliberate: rows are the declared enum in DECLARED order (so two runs of the
+ * same suite are diffable line by line) plus {@link NO_EXPECTATION} only when some cell declares no
+ * expectation; columns are the declared enum plus {@link UNRECOGNIZED_LABEL} and/or
+ * {@link NO_EXPECTATION} only when some cell actually produced one. Extra buckets never appear
+ * speculatively, and never fail to appear when they are populated.
+ *
+ * `counted + excluded === cells.length`, always — asserted in the unit suite, because a matrix that
+ * silently drops the cells it could not place reads as "covered everything" when it didn't.
+ */
+export function buildConfusionMatrix(cells, dimension, declared) {
+    const expectedOf = (cell) => dimension === 'label' ? cell.expectedLabel : cell.expectedAction;
+    const actualOf = (cell) => dimension === 'label' ? cell.actualLabel : cell.actualAction;
+    const scored = cells.filter((cell) => cell.scored);
+    const excluded = cells.length - scored.length;
+    const rowSet = new Set(declared);
+    const columnSet = new Set(declared);
+    for (const cell of scored) {
+        rowSet.add(expectedOf(cell) ?? NO_EXPECTATION);
+        columnSet.add(actualOf(cell) ?? NO_EXPECTATION);
+    }
+    // Preserve DECLARED order first, then any extra bucket in a stable order, so the rendered matrix
+    // is diffable across runs.
+    const order = (present) => [
+        ...declared.filter((value) => present.has(value)),
+        ...[UNRECOGNIZED_LABEL, NO_EXPECTATION].filter((value) => present.has(value)),
+    ];
+    // Declared values always keep their row/column even at zero — an enum value that never appears is
+    // itself a finding (the QA-5 "the caution tier was never emitted" result), so it must be visible.
+    for (const value of declared) {
+        rowSet.add(value);
+        columnSet.add(value);
+    }
+    const rows = order(rowSet);
+    const columns = order(columnSet);
+    const counts = {};
+    for (const row of rows) {
+        counts[row] = {};
+        for (const column of columns)
+            counts[row][column] = 0;
+    }
+    for (const cell of scored) {
+        const row = expectedOf(cell) ?? NO_EXPECTATION;
+        const column = actualOf(cell) ?? NO_EXPECTATION;
+        counts[row][column] += 1;
+    }
+    return { dimension, rows, columns, counts, counted: scored.length, excluded };
+}
+/** Every tag any cell declares, sorted — the per-tag axis for matrices and metrics. */
+export function collectTags(cells) {
+    const tags = new Set();
+    for (const cell of cells)
+        for (const tag of cell.tags)
+            tags.add(tag);
+    return [...tags].sort();
+}
+/**
+ * Which enum a `classification` spec declares for a dimension. `actions` is `[]` when the suite has
+ * no action dimension, which is what makes an `expect_action` assertion a parse error rather than a
+ * silently ungradeable one.
+ */
+export function declaredValues(spec, dimension) {
+    return dimension === 'label' ? spec.labels : spec.actions;
+}
+//# sourceMappingURL=classification.js.map

package/dist/classification.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classification.js","sourceRoot":"","sources":["../src/classification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAO9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAEjF;;;;;;;;GAQG;AAEH;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,0BAA0B,CACxC,MAA0B,EAC1B,SAAkC,EAClC,OAAiB;IAEjB,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,kBAAkB,CAAC;IACjD,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACvD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,UAAU,CAAC,CAAC;IAChF,OAAO,GAAG,IAAI,kBAAkB,CAAC;AACnC,CAAC;AAED;;sGAEsG;AACtG,MAAM,UAAU,OAAO,CACrB,MAA0B,EAC1B,SAAkC;IAElC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAE/C,8FAA8F;IAC9F,kGAAkG;IAClG,gFAAgF;IAChF,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/D,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACtE,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED;oBACoB;AACpB,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACtB,kFAAkF;IAClF,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACzF,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;AAC5B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAuB,EACvB,SAA6B,EAC7B,QAAkB;IAElB,MAAM,UAAU,GAAG,CAAC,IAAoB,EAAsB,EAAE,CAC9D,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;IACnE,MAAM,QAAQ,GAAG,CAAC,IAAoB,EAAsB,EAAE,CAC5D,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC;QAC/C,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC;IAClD,CAAC;IACD,iGAAiG;IACjG,2BAA2B;IAC3B,MAAM,KAAK,GAAG,CAAC,OAAoB,EAAY,EAAE,CAAC;QAChD,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjD,GAAG,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KAC9E,CAAC;IACF,kGAAkG;IAClG,kGAAkG;IAClG,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClB,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IAEjC,MAAM,MAAM,GAA2C,EAAE,CAAC;IAC1D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC;QAC/C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;AAChF,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,IAA4B,EAC5B,SAA6B;IAE7B,OAAO,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAC5D,CAAC"}

package/dist/classificationRender.d.ts

@@ -0,0 +1,24 @@
+import type { EvalClassificationReport, EvalConfusionMatrix, EvalMetricResult } from '#src/classificationTypes.js';
+/**
+ * BATCH-25 — render a {@link EvalClassificationReport} as plain lines.
+ *
+ * A pure string builder, deliberately separate from the reporter that prints it: the console
+ * rendering is then unit-testable without mocking `consoleUtils`, and a future reporter (JUnit,
+ * TeamCity, a third-party one) can emit the same block without reimplementing the layout.
+ *
+ * Every function here is total over the report — there is no "…and 12 more" anywhere. A renderer
+ * that truncates is a silent cap, and a truncated matrix reads as a complete one.
+ */
+/** Rendering options. */
+export interface RenderClassificationOptions {
+    /** Suppress the per-tag confusion matrices (the per-tag METRIC rows always stay). Set for a
+     * sweep cell, where N full blocks would bury the comparison table that is the point of sweeping.
+     * The suppressed matrices are still written to that cell's `results.json`. */
+    compact?: boolean;
+}
+/** Render the whole report: coverage, matrices, metrics, warnings, gate verdict. */
+export declare function renderClassificationReport(report: EvalClassificationReport, options?: RenderClassificationOptions): string[];
+/** Render one confusion matrix as a fixed-width grid: rows = expected, columns = actual. */
+export declare function renderConfusionMatrix(matrix: EvalConfusionMatrix, title: string): string[];
+/** Render one metric: headline value, gate verdict, per-tag sub-scores, then its warnings. */
+export declare function renderMetric(metric: EvalMetricResult, tags: string[]): string[];