@gaunt-sloth/batch 2.0.0-alpha.19

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2025-present Andrew Kondratev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/dist/BatchRunner.d.ts

@@ -0,0 +1,18 @@
+import { type BatchRunnerOptions, type BatchSummary, type CellResult, type MatrixCell } from '#src/types.js';
+/**
+ * Run every cell of the matrix through the injected {@link RunCellFn}, capping in-flight work at
+ * `concurrency` and retrying a failed cell up to `retry` times.
+ *
+ * Isolation: cells are independent by construction — nothing here shares state between them beyond
+ * the caller-supplied `runCell` closure, matching the "N isolated model calls" framing batch is
+ * scoped around (docs/batch-mechanism-vs-judgment.md). A cell throwing (rather than resolving
+ * `{ ok: false }`) is caught and recorded as a failure like any other — one bad cell must never
+ * take down the whole batch run or the other in-flight cells (the exit-code contract: `gth batch`
+ * exits 0 iff the cells ran at all, regardless of how many of them failed).
+ *
+ * Returns results in the same order as `cells` (not completion order), so callers can zip them back
+ * up with the matrix.
+ */
+export declare function runBatchMatrix(cells: MatrixCell[], options: BatchRunnerOptions): Promise<CellResult[]>;
+/** Build the lightweight aggregate "flake report" (pass/fail counts) from the per-cell results. */
+export declare function buildBatchSummary(results: CellResult[]): BatchSummary;

package/dist/BatchRunner.js

@@ -0,0 +1,100 @@
+import { DEFAULT_CONCURRENCY, } from '#src/types.js';
+/**
+ * Run every cell of the matrix through the injected {@link RunCellFn}, capping in-flight work at
+ * `concurrency` and retrying a failed cell up to `retry` times.
+ *
+ * Isolation: cells are independent by construction — nothing here shares state between them beyond
+ * the caller-supplied `runCell` closure, matching the "N isolated model calls" framing batch is
+ * scoped around (docs/batch-mechanism-vs-judgment.md). A cell throwing (rather than resolving
+ * `{ ok: false }`) is caught and recorded as a failure like any other — one bad cell must never
+ * take down the whole batch run or the other in-flight cells (the exit-code contract: `gth batch`
+ * exits 0 iff the cells ran at all, regardless of how many of them failed).
+ *
+ * Returns results in the same order as `cells` (not completion order), so callers can zip them back
+ * up with the matrix.
+ */
+export async function runBatchMatrix(cells, options) {
+    const concurrency = normalizeConcurrency(options.concurrency);
+    const retry = normalizeRetry(options.retry);
+    const results = new Array(cells.length);
+    let nextIndex = 0;
+    const worker = async () => {
+        for (;;) {
+            const i = nextIndex++;
+            if (i >= cells.length)
+                return;
+            results[i] = await runCellWithRetry(cells[i], options.runCell, retry);
+        }
+    };
+    const workerCount = Math.min(concurrency, cells.length);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    return results;
+}
+async function runCellWithRetry(cell, runCell, retry) {
+    const startedAt = Date.now();
+    let attempts = 0;
+    // Always at least one attempt; `retry` additional attempts on failure.
+    for (;;) {
+        attempts++;
+        try {
+            const outcome = await runCell(cell);
+            if (outcome.ok || attempts > retry) {
+                return {
+                    ...outcome,
+                    id: cell.id,
+                    model: cell.model,
+                    inputIndex: cell.inputIndex,
+                    inputRow: cell.inputRow,
+                    durationMs: Date.now() - startedAt,
+                    retries: attempts - 1,
+                };
+            }
+            // Failed but retries remain: loop around for another attempt.
+        }
+        catch (error) {
+            if (attempts > retry) {
+                return {
+                    ok: false,
+                    error: error instanceof Error ? error.message : String(error),
+                    id: cell.id,
+                    model: cell.model,
+                    inputIndex: cell.inputIndex,
+                    inputRow: cell.inputRow,
+                    durationMs: Date.now() - startedAt,
+                    retries: attempts - 1,
+                };
+            }
+            // Exception with retries remaining: loop around for another attempt.
+        }
+    }
+}
+function normalizeConcurrency(concurrency) {
+    if (concurrency === undefined || !Number.isFinite(concurrency) || concurrency < 1) {
+        return DEFAULT_CONCURRENCY;
+    }
+    return Math.floor(concurrency);
+}
+function normalizeRetry(retry) {
+    if (retry === undefined || !Number.isFinite(retry) || retry < 0) {
+        return 0;
+    }
+    return Math.floor(retry);
+}
+/** Build the lightweight aggregate "flake report" (pass/fail counts) from the per-cell results. */
+export function buildBatchSummary(results) {
+    const cells = results.map((r) => ({
+        id: r.id,
+        model: r.model,
+        inputIndex: r.inputIndex,
+        ok: r.ok,
+        retries: r.retries,
+    }));
+    const passed = cells.filter((c) => c.ok).length;
+    return {
+        total: cells.length,
+        passed,
+        failed: cells.length - passed,
+        cells,
+    };
+}
+//# sourceMappingURL=BatchRunner.js.map

package/dist/BatchRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BatchRunner.js","sourceRoot":"","sources":["../src/BatchRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,GAMpB,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAmB,EACnB,OAA2B;IAE3B,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAEtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,SAAS,CAAC;YACR,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEvE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAgB,EAChB,OAAkB,EAClB,KAAa;IAEb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,uEAAuE;IACvE,SAAS,CAAC;QACR,QAAQ,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,EAAE,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACnC,OAAO;oBACL,GAAG,OAAO;oBACV,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,8DAA8D;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACrB,OAAO;oBACL,EAAE,EAAE,KAAK;oBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oBAClC,OAAO,EAAE,QAAQ,GAAG,CAAC;iBACtB,CAAC;YACJ,CAAC;YACD,qEAAqE;QACvE,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,WAA+B;IAC3D,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAClF,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,OAAqB;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,MAAM;QACN,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM;QAC7B,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/deterministicChecks.d.ts

@@ -0,0 +1,16 @@
+import type { DeterministicCheckResult, EvalCase } from '#src/evalTypes.js';
+/**
+ * Case-insensitive substring checks over an SUT answer. Ported, not reinvented, from the field
+ * user's proven `deterministic()` function (`docs/batch-eval-user-requirements.md` Appendix A):
+ *
+ * - `mustContain` — every entry must appear (case-insensitive substring); each miss is reported.
+ * - `mustNotContain` — no entry may appear; each hit is reported.
+ * - `shouldContainAny` — at least one entry must appear; reported as a single combined failure
+ *   (not one per missing option — the check is "at least one", so there is only one way to fail
+ *   it) when none do.
+ *
+ * A case with all three arrays empty trivially passes (no checks to fail) — the suite parser
+ * (`#src/evalSuite.js`) is what enforces that a case has *some* check or a judge rubric; this
+ * function itself has no opinion on that.
+ */
+export declare function runDeterministicChecks(answer: string, evalCase: Pick<EvalCase, 'mustContain' | 'mustNotContain' | 'shouldContainAny'>): DeterministicCheckResult;

package/dist/deterministicChecks.js

@@ -0,0 +1,34 @@
+/**
+ * Case-insensitive substring checks over an SUT answer. Ported, not reinvented, from the field
+ * user's proven `deterministic()` function (`docs/batch-eval-user-requirements.md` Appendix A):
+ *
+ * - `mustContain` — every entry must appear (case-insensitive substring); each miss is reported.
+ * - `mustNotContain` — no entry may appear; each hit is reported.
+ * - `shouldContainAny` — at least one entry must appear; reported as a single combined failure
+ *   (not one per missing option — the check is "at least one", so there is only one way to fail
+ *   it) when none do.
+ *
+ * A case with all three arrays empty trivially passes (no checks to fail) — the suite parser
+ * (`#src/evalSuite.js`) is what enforces that a case has *some* check or a judge rubric; this
+ * function itself has no opinion on that.
+ */
+export function runDeterministicChecks(answer, evalCase) {
+    const text = answer.toLowerCase();
+    const failures = [];
+    for (const needle of evalCase.mustContain) {
+        if (!text.includes(needle.toLowerCase())) {
+            failures.push(`missing "${needle}"`);
+        }
+    }
+    for (const needle of evalCase.mustNotContain) {
+        if (text.includes(needle.toLowerCase())) {
+            failures.push(`forbidden "${needle}"`);
+        }
+    }
+    if (evalCase.shouldContainAny.length > 0 &&
+        !evalCase.shouldContainAny.some((needle) => text.includes(needle.toLowerCase()))) {
+        failures.push(`none of [${evalCase.shouldContainAny.join(' | ')}]`);
+    }
+    return { passed: failures.length === 0, failures };
+}
+//# sourceMappingURL=deterministicChecks.js.map

package/dist/deterministicChecks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministicChecks.js","sourceRoot":"","sources":["../src/deterministicChecks.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAc,EACd,QAA+E;IAE/E,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,YAAY,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,cAAc,MAAM,GAAG,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,IACE,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACpC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,EAChF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,YAAY,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;AACrD,CAAC"}

package/dist/evalOutput.d.ts

@@ -0,0 +1,11 @@
+import type { EvalSuiteSummary } from '#src/evalTypes.js';
+/**
+ * Write one structured JSON record per case (`<id>.json`) plus one aggregate `results.json`
+ * (suite totals + every case's verdict/checks/judge/reasons) into `outputDir`. Creates `outputDir`
+ * (and any missing parents) if it doesn't exist. Mirrors BATCH-1's `writeBatchOutput` (`#src/
+ * output.js`) — same convention, applied to eval's richer per-case shape.
+ *
+ * Pure I/O, deliberately separate from {@link ../evalRunner.js}'s `runEvalSuite`: the runner never
+ * touches the filesystem, so unit tests can exercise grading logic without a tmp dir.
+ */
+export declare function writeEvalOutput(outputDir: string, summary: EvalSuiteSummary): void;

package/dist/evalOutput.js

@@ -0,0 +1,19 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+/**
+ * Write one structured JSON record per case (`<id>.json`) plus one aggregate `results.json`
+ * (suite totals + every case's verdict/checks/judge/reasons) into `outputDir`. Creates `outputDir`
+ * (and any missing parents) if it doesn't exist. Mirrors BATCH-1's `writeBatchOutput` (`#src/
+ * output.js`) — same convention, applied to eval's richer per-case shape.
+ *
+ * Pure I/O, deliberately separate from {@link ../evalRunner.js}'s `runEvalSuite`: the runner never
+ * touches the filesystem, so unit tests can exercise grading logic without a tmp dir.
+ */
+export function writeEvalOutput(outputDir, summary) {
+    mkdirSync(outputDir, { recursive: true });
+    for (const result of summary.cases) {
+        writeFileSync(join(outputDir, `${result.id}.json`), `${JSON.stringify(result, null, 2)}\n`);
+    }
+    writeFileSync(join(outputDir, 'results.json'), `${JSON.stringify(summary, null, 2)}\n`);
+}
+//# sourceMappingURL=evalOutput.js.map

package/dist/evalOutput.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalOutput.js","sourceRoot":"","sources":["../src/evalOutput.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,OAAyB;IAC1E,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1F,CAAC"}

package/dist/evalRunner.d.ts

@@ -0,0 +1,25 @@
+import type { RunCellFn } from '#src/types.js';
+import type { EvalSuite, EvalSuiteSummary, JudgeFn } from '#src/evalTypes.js';
+/** Options for {@link runEvalSuite}. */
+export interface RunEvalSuiteOptions {
+    /** The injectable per-case run function — same seam as BATCH-1's `RunCellFn`, adapted to send
+     * each case's `prompt` through the SUT (production wiring: `evalCommand.ts`, reusing
+     * `batchCommand.ts`'s `buildProductionRunCell`; tests inject a fake). */
+    runCell: RunCellFn;
+    /** The injectable judge function. Only consulted for cases that declare a `judgeRubric`;
+     * omitted entirely = every judge-rubric case fails with a "no judge configured" reason (this
+     * should not happen in production wiring, where `evalCommand.ts` always supplies one, but the
+     * runner degrades safely rather than throwing if it's left out). */
+    judge?: JudgeFn;
+    /** Max in-flight cases — reuses BATCH-1's `runBatchMatrix` pool (`DEFAULT_CONCURRENCY` when
+     * omitted); no second concurrency mechanism is introduced for eval. */
+    concurrency?: number;
+}
+/**
+ * Run every case in the suite through the SUT ({@link RunEvalSuiteOptions.runCell}, pooled via
+ * BATCH-1's `runBatchMatrix`), then grade each answer with deterministic checks and (when the case
+ * declares a rubric) the judge. A case PASSES iff its deterministic checks pass AND (it has no
+ * judge rubric OR the judge's rate is at/above the case's `passThreshold`) — ported from the field
+ * user's proven harness semantics (`docs/batch-eval-user-requirements.md` Appendix A).
+ */
+export declare function runEvalSuite(suite: EvalSuite, options: RunEvalSuiteOptions): Promise<EvalSuiteSummary>;

package/dist/evalRunner.js

@@ -0,0 +1,100 @@
+import { runBatchMatrix } from '#src/BatchRunner.js';
+import { runDeterministicChecks } from '#src/deterministicChecks.js';
+/**
+ * Build one {@link MatrixCell} per case, `content` = the case's `prompt`. Unlike BATCH-1's
+ * `buildMatrix` (a models × rows cross product), eval has exactly one cell per case — so cells are
+ * built directly rather than via `buildMatrix`/`bindCellContent`, which would force an artificial
+ * single-row/single-model matrix per case for no benefit (no `{{field}}` interpolation applies to
+ * an eval case's literal `prompt`).
+ *
+ * `cell.id` is the case's own `id` (not a `cell-<m>-<r>` index) so per-case JSON output filenames
+ * read naturally (`<case-id>.json` — see `#src/evalOutput.js`) and results are trivially traceable
+ * back to the authored case. `parseEvalSuite` already rejects duplicate case ids, so this can't
+ * collide.
+ */
+function buildEvalCells(cases) {
+    return cases.map((evalCase, index) => ({
+        id: evalCase.id,
+        modelIndex: 0,
+        inputIndex: index,
+        content: evalCase.prompt,
+    }));
+}
+/**
+ * Run every case in the suite through the SUT ({@link RunEvalSuiteOptions.runCell}, pooled via
+ * BATCH-1's `runBatchMatrix`), then grade each answer with deterministic checks and (when the case
+ * declares a rubric) the judge. A case PASSES iff its deterministic checks pass AND (it has no
+ * judge rubric OR the judge's rate is at/above the case's `passThreshold`) — ported from the field
+ * user's proven harness semantics (`docs/batch-eval-user-requirements.md` Appendix A).
+ */
+export async function runEvalSuite(suite, options) {
+    const cells = buildEvalCells(suite.cases);
+    const cellResults = await runBatchMatrix(cells, {
+        runCell: options.runCell,
+        concurrency: options.concurrency,
+    });
+    const casesById = new Map(suite.cases.map((evalCase) => [evalCase.id, evalCase]));
+    const results = [];
+    for (const cellResult of cellResults) {
+        const evalCase = casesById.get(cellResult.id);
+        /* istanbul ignore next -- cell ids are derived 1:1 from case ids by buildEvalCells above */
+        if (!evalCase)
+            continue;
+        results.push(await gradeCase(evalCase, cellResult, options.judge));
+    }
+    const passed = results.filter((result) => result.verdict === 'PASS').length;
+    return { total: results.length, passed, failed: results.length - passed, cases: results };
+}
+async function gradeCase(evalCase, cellResult, judge) {
+    const base = {
+        id: evalCase.id,
+        passThreshold: evalCase.passThreshold,
+        answer: cellResult.answer,
+        tokensInput: cellResult.tokensInput,
+        tokensOutput: cellResult.tokensOutput,
+        tools: cellResult.tools,
+        durationMs: cellResult.durationMs,
+    };
+    if (!cellResult.ok) {
+        return {
+            ...base,
+            verdict: 'FAIL',
+            sutOk: false,
+            reasons: [cellResult.error ? `SUT run failed: ${cellResult.error}` : 'SUT run failed.'],
+        };
+    }
+    const answer = cellResult.answer ?? '';
+    const checks = runDeterministicChecks(answer, evalCase);
+    const reasons = [...checks.failures];
+    let judgeOutcome;
+    let judgePassed = true;
+    if (evalCase.judgeRubric) {
+        if (!judge) {
+            judgeOutcome = { attempted: false, ok: false, error: 'No judge configured for this suite.' };
+            judgePassed = false;
+            reasons.push('judge: not configured');
+        }
+        else {
+            judgeOutcome = await judge(answer, evalCase.judgeRubric);
+            if (!judgeOutcome.ok || !judgeOutcome.verdict) {
+                judgePassed = false;
+                reasons.push(`judge error: ${judgeOutcome.error ?? 'unknown error'}`);
+            }
+            else if (judgeOutcome.verdict.rate < evalCase.passThreshold) {
+                judgePassed = false;
+                reasons.push(`judge rate ${judgeOutcome.verdict.rate}/10 below threshold ${evalCase.passThreshold}` +
+                    (judgeOutcome.verdict.reason ? `: ${judgeOutcome.verdict.reason}` : ''));
+            }
+        }
+    }
+    const verdict = checks.passed && judgePassed ? 'PASS' : 'FAIL';
+    return {
+        ...base,
+        verdict,
+        sutOk: true,
+        checks,
+        judge: judgeOutcome,
+        reasons,
+    };
+}
+//# sourceMappingURL=evalRunner.js.map

package/dist/evalRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AA2BrE;;;;;;;;;;;GAWG;AACH,SAAS,cAAc,CAAC,KAAiB;IACvC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACrC,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE,QAAQ,CAAC,MAAM;KACzB,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC9C,4FAA4F;QAC5F,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAkB,EAClB,UAAsB,EACtB,KAA0B;IAE1B,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAErC,IAAI,YAAsC,CAAC;IAC3C,IAAI,WAAW,GAAG,IAAI,CAAC;IACvB,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,YAAY,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;YAC7F,WAAW,GAAG,KAAK,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YACzD,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9C,WAAW,GAAG,KAAK,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;YACxE,CAAC;iBAAM,IAAI,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAC9D,WAAW,GAAG,KAAK,CAAC;gBACpB,OAAO,CAAC,IAAI,CACV,cAAc,YAAY,CAAC,OAAO,CAAC,IAAI,uBAAuB,QAAQ,CAAC,aAAa,EAAE;oBACpF,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAoB,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhF,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM;QACN,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/evalSuite.d.ts

@@ -0,0 +1,24 @@
+import type { EvalSuite } from '#src/evalTypes.js';
+/**
+ * Parse and validate an eval suite YAML document into a normalized {@link EvalSuite}.
+ *
+ * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
+ * - Malformed YAML.
+ * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
+ * - `target.type` other than `"gth-agent"` — pluggable CLI/HTTP targets are out of scope for this
+ *   task (see the CLI surface doc's future `target: {type: cli|http}` sketch).
+ * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
+ *   switch is the same `--identities` direction the brief scopes out; this task's target is always
+ *   whatever profile `gth eval` itself was invoked under (`-i/--identity-profile`, if any).
+ * - A case `id` containing anything other than alphanumerics, dashes, underscores, or dots (case
+ *   ids double as output filenames — see `#src/evalOutput.js` — so a path separator or traversal
+ *   sequence like `../../etc/passwd` is rejected here, not sanitized).
+ * - A duplicate case `id` (case ids double as output filenames — see `#src/evalOutput.js`).
+ * - A case with **no** deterministic checks (`must_contain`/`must_not_contain`/
+ *   `should_contain_any` all absent/empty) **and no** `judge` rubric: nothing would ever grade it,
+ *   which is a suite-authoring bug, not a case that trivially passes.
+ *
+ * @param yamlText Raw suite file content.
+ * @param sourcePath Optional path, only used to make error messages more actionable.
+ */
+export declare function parseEvalSuite(yamlText: string, sourcePath?: string): EvalSuite;

package/dist/evalSuite.js

@@ -0,0 +1,129 @@
+import { parse as parseYaml } from 'yaml';
+import * as z from 'zod';
+import { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+/**
+ * Raw suite-file shape (snake_case, as authored) — see the BATCH-2 Task 2 brief and
+ * `docs/batch-eval-user-requirements.md`'s YAML sketch, trimmed to the single-`prompt`,
+ * no-identities, no-fixtures subset this task implements:
+ *
+ * ```yaml
+ * target: { type: gth-agent, profile: default }
+ * defaults: { pass_threshold: 6 }
+ * cases:
+ *   - id: some-case-id
+ *     prompt: "the user message to send"
+ *     must_contain: [ "foo", "bar" ]
+ *     must_not_contain: [ "baz" ]
+ *     should_contain_any: [ "x", "y" ]
+ *     judge: "Answers with a ranked summary and correctly formatted values."
+ *     pass_threshold: 7
+ * ```
+ */
+const RawCaseSchema = z.object({
+    id: z
+        .string()
+        .min(1, 'case id must be a non-empty string')
+        .regex(/^[\w.-]+$/, 'case id must be a valid filename (alphanumeric, dashes, underscores, dots) — case ids ' +
+        'double as output filenames, so path separators and other special characters are rejected'),
+    prompt: z.string().min(1, 'case prompt must be a non-empty string'),
+    must_contain: z.array(z.string()).optional(),
+    must_not_contain: z.array(z.string()).optional(),
+    should_contain_any: z.array(z.string()).optional(),
+    judge: z.string().optional(),
+    pass_threshold: z.number().min(0).max(10).optional(),
+});
+const RawSuiteSchema = z.object({
+    target: z.object({
+        type: z.string(),
+        profile: z.string().optional(),
+    }),
+    defaults: z
+        .object({
+        pass_threshold: z.number().min(0).max(10).optional(),
+    })
+        .optional(),
+    cases: z.array(RawCaseSchema).min(1, 'suite must declare at least one case'),
+});
+/**
+ * Parse and validate an eval suite YAML document into a normalized {@link EvalSuite}.
+ *
+ * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
+ * - Malformed YAML.
+ * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
+ * - `target.type` other than `"gth-agent"` — pluggable CLI/HTTP targets are out of scope for this
+ *   task (see the CLI surface doc's future `target: {type: cli|http}` sketch).
+ * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
+ *   switch is the same `--identities` direction the brief scopes out; this task's target is always
+ *   whatever profile `gth eval` itself was invoked under (`-i/--identity-profile`, if any).
+ * - A case `id` containing anything other than alphanumerics, dashes, underscores, or dots (case
+ *   ids double as output filenames — see `#src/evalOutput.js` — so a path separator or traversal
+ *   sequence like `../../etc/passwd` is rejected here, not sanitized).
+ * - A duplicate case `id` (case ids double as output filenames — see `#src/evalOutput.js`).
+ * - A case with **no** deterministic checks (`must_contain`/`must_not_contain`/
+ *   `should_contain_any` all absent/empty) **and no** `judge` rubric: nothing would ever grade it,
+ *   which is a suite-authoring bug, not a case that trivially passes.
+ *
+ * @param yamlText Raw suite file content.
+ * @param sourcePath Optional path, only used to make error messages more actionable.
+ */
+export function parseEvalSuite(yamlText, sourcePath) {
+    const suffix = sourcePath ? ` (${sourcePath})` : '';
+    let raw;
+    try {
+        raw = parseYaml(yamlText);
+    }
+    catch (error) {
+        throw new Error(`Failed to parse eval suite YAML${suffix}: ` +
+            (error instanceof Error ? error.message : String(error)));
+    }
+    const parsed = RawSuiteSchema.safeParse(raw);
+    if (!parsed.success) {
+        const issues = parsed.error.issues
+            .map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`)
+            .join('; ');
+        throw new Error(`Invalid eval suite${suffix}: ${issues}`);
+    }
+    const data = parsed.data;
+    if (data.target.type !== 'gth-agent') {
+        throw new Error(`Invalid eval suite${suffix}: unsupported target.type "${data.target.type}" — this version ` +
+            'of `gth eval` only supports "gth-agent" (pluggable CLI/HTTP targets are future scope).');
+    }
+    if (data.target.profile !== undefined && data.target.profile !== 'default') {
+        throw new Error(`Invalid eval suite${suffix}: unsupported target.profile "${data.target.profile}" — ` +
+            'per-profile/identity targets are not supported yet (see --identities in ' +
+            'docs/batch-eval-cli-surface.md); omit target.profile or set it to "default".');
+    }
+    const suiteDefaultThreshold = data.defaults?.pass_threshold ?? DEFAULT_EVAL_PASS_THRESHOLD;
+    const seenIds = new Set();
+    const cases = data.cases.map((rawCase, index) => {
+        if (seenIds.has(rawCase.id)) {
+            throw new Error(`Invalid eval suite${suffix}: duplicate case id "${rawCase.id}".`);
+        }
+        seenIds.add(rawCase.id);
+        const mustContain = rawCase.must_contain ?? [];
+        const mustNotContain = rawCase.must_not_contain ?? [];
+        const shouldContainAny = rawCase.should_contain_any ?? [];
+        const hasChecks = mustContain.length > 0 || mustNotContain.length > 0 || shouldContainAny.length > 0;
+        const judgeRubric = rawCase.judge?.trim();
+        const hasJudge = !!judgeRubric;
+        if (!hasChecks && !hasJudge) {
+            throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) has no deterministic ` +
+                'checks and no judge rubric — a case must declare at least one of must_contain / ' +
+                'must_not_contain / should_contain_any, or a judge rubric.');
+        }
+        return {
+            id: rawCase.id,
+            prompt: rawCase.prompt,
+            mustContain,
+            mustNotContain,
+            shouldContainAny,
+            judgeRubric: hasJudge ? judgeRubric : undefined,
+            passThreshold: rawCase.pass_threshold ?? suiteDefaultThreshold,
+        };
+    });
+    return {
+        target: { type: 'gth-agent', profile: data.target.profile },
+        cases,
+    };
+}
+//# sourceMappingURL=evalSuite.js.map

package/dist/evalSuite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSuite.js","sourceRoot":"","sources":["../src/evalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAGhE;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAC5C,KAAK,CACJ,WAAW,EACX,wFAAwF;QACtF,0FAA0F,CAC7F;IACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wCAAwC,CAAC;IACnE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC5C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC/B,CAAC;IACF,QAAQ,EAAE,CAAC;SACR,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;KACrD,CAAC;SACD,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC7E,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,UAAmB;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,kCAAkC,MAAM,IAAI;YAC1C,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACvE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAEzB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8BAA8B,IAAI,CAAC,MAAM,CAAC,IAAI,mBAAmB;YAC1F,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,iCAAiC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;YACnF,0EAA0E;YAC1E,8EAA8E,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,IAAI,2BAA2B,CAAC;IAE3F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,wBAAwB,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAExB,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;QAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GACb,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;QACrF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC;QAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,yBAAyB;gBACxF,kFAAkF;gBAClF,2DAA2D,CAC9D,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW;YACX,cAAc;YACd,gBAAgB;YAChB,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YAC/C,aAAa,EAAE,OAAO,CAAC,cAAc,IAAI,qBAAqB;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAC3D,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/evalTypes.d.ts

@@ -0,0 +1,98 @@
+/**
+ * @packageDocumentation
+ * BATCH-2 — the shapes for `gth eval`: a parsed suite/case, deterministic-check results, the
+ * judge's verdict, and per-case/suite outcomes. Deliberately separate from {@link ../types.js}
+ * (BATCH-1's cell/outcome shapes), which documents itself as scoped to "cells and outcomes" only
+ * — eval's shapes layer on top of (not into) that file.
+ */
+/** The 0-10 judge scale's default pass threshold, matching `review`'s own (unexported)
+ * `DEFAULT_PASS_THRESHOLD` in `packages/review/src/middleware/reviewRateMiddleware.ts` — same
+ * scale, same default, so a user who knows `review`'s threshold semantics already knows eval's.
+ * Kept as our own constant (not imported) since review's is private to that module and coupled to
+ * middleware/tool-call/artifact-store plumbing that doesn't fit a plain structured-output call. */
+export declare const DEFAULT_EVAL_PASS_THRESHOLD = 6;
+/** One case's target — this task only supports `gth-agent` against the run's own resolved config
+ * (no `--identities`, no pluggable CLI/HTTP targets — see the BATCH-2 Task 2 brief). */
+export interface EvalTarget {
+    type: 'gth-agent';
+    /** Suite-level profile hint. Only `undefined`/`'default'` is accepted this task (see
+     * {@link ../evalSuite.js}'s `parseEvalSuite`) — per-case/per-identity profile switching is
+     * `--identities` scope, not this task's. */
+    profile?: string;
+}
+/** One case parsed and normalized from suite YAML — snake_case YAML keys become camelCase here,
+ * arrays default to `[]` (not `undefined`) so callers never need an existence check, and
+ * `passThreshold` is pre-resolved (case override ?? suite `defaults.pass_threshold` ??
+ * {@link DEFAULT_EVAL_PASS_THRESHOLD}). */
+export interface EvalCase {
+    id: string;
+    prompt: string;
+    mustContain: string[];
+    mustNotContain: string[];
+    shouldContainAny: string[];
+    /** The judge rubric, when present and non-blank. `undefined` = no judge for this case. */
+    judgeRubric?: string;
+    passThreshold: number;
+}
+/** A fully parsed and validated suite — see {@link ../evalSuite.js}'s `parseEvalSuite`. */
+export interface EvalSuite {
+    target: EvalTarget;
+    cases: EvalCase[];
+}
+/** The result of running one case's answer through its deterministic checks. */
+export interface DeterministicCheckResult {
+    passed: boolean;
+    /** Human-readable failure reasons, e.g. `missing "foo"` / `forbidden "baz"` /
+     * `none of [x | y]` — the exact message shapes from the field user's proven harness
+     * (`docs/batch-eval-user-requirements.md` Appendix A's `deterministic()`). Empty when passed. */
+    failures: string[];
+}
+/** The judge's structured verdict on one case's answer — matches `review`'s `RateSchema` shape
+ * (0-10 `rate` + a reason string) for UX consistency, see {@link ../judge.js}. */
+export interface JudgeVerdict {
+    rate: number;
+    reason: string;
+}
+/** The outcome of attempting to grade one case's answer with the judge. */
+export interface JudgeOutcome {
+    /** `true` once a judge call was actually made (vs. skipped because no judge was configured). */
+    attempted: boolean;
+    /** `true` when the judge produced a usable {@link JudgeVerdict}. `false` on any error, timeout,
+     * or unparseable output — per the BATCH-2 brief, a judge that can't produce a verdict FAILS the
+     * case (there is no human to escalate to here, unlike EXT-10's shell-safety judge). */
+    ok: boolean;
+    verdict?: JudgeVerdict;
+    /** Set when `ok` is `false`: why the judge didn't produce a verdict. */
+    error?: string;
+}
+/** Injectable "grade one answer against one rubric" function — the seam that lets
+ * {@link ../evalRunner.js}'s `runEvalSuite` be fully unit tested without any real LLM call, mirror
+ * of BATCH-1's `RunCellFn`. The production wiring (`evalCommand.ts`) adapts `judgeEvalCase` to
+ * this shape; tests inject a fake that resolves/fails as needed. */
+export type JudgeFn = (answer: string, rubric: string) => Promise<JudgeOutcome>;
+/** One case's full graded outcome, as written to `<id>.json` and summarized in `results.json`. */
+export interface EvalCaseResult {
+    id: string;
+    verdict: 'PASS' | 'FAIL';
+    passThreshold: number;
+    /** Whether the SUT run itself completed without error (mirrors `CellRunOutcome.ok`). `false`
+     * means the case FAILs outright — there is no answer text to check or grade. */
+    sutOk: boolean;
+    answer?: string;
+    tokensInput?: number;
+    tokensOutput?: number;
+    tools?: string[];
+    durationMs: number;
+    checks?: DeterministicCheckResult;
+    judge?: JudgeOutcome;
+    /** Every reason the case FAILed (deterministic check failures, judge-below-threshold, judge
+     * error, SUT failure). Empty when `verdict` is `PASS`. */
+    reasons: string[];
+}
+/** The suite-level aggregate written to `results.json` — `gth eval` exits 0 iff `failed === 0`. */
+export interface EvalSuiteSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    cases: EvalCaseResult[];
+}

package/dist/evalTypes.js

@@ -0,0 +1,14 @@
+/**
+ * @packageDocumentation
+ * BATCH-2 — the shapes for `gth eval`: a parsed suite/case, deterministic-check results, the
+ * judge's verdict, and per-case/suite outcomes. Deliberately separate from {@link ../types.js}
+ * (BATCH-1's cell/outcome shapes), which documents itself as scoped to "cells and outcomes" only
+ * — eval's shapes layer on top of (not into) that file.
+ */
+/** The 0-10 judge scale's default pass threshold, matching `review`'s own (unexported)
+ * `DEFAULT_PASS_THRESHOLD` in `packages/review/src/middleware/reviewRateMiddleware.ts` — same
+ * scale, same default, so a user who knows `review`'s threshold semantics already knows eval's.
+ * Kept as our own constant (not imported) since review's is private to that module and coupled to
+ * middleware/tool-call/artifact-store plumbing that doesn't fit a plain structured-output call. */
+export const DEFAULT_EVAL_PASS_THRESHOLD = 6;
+//# sourceMappingURL=evalTypes.js.map