@gaunt-sloth/batch 2.0.0-alpha.19 → 2.0.0-alpha.21

package/dist/evalRunner.js

@@ -1,53 +1,225 @@
 import { runBatchMatrix } from '#src/BatchRunner.js';
 import { runDeterministicChecks } from '#src/deterministicChecks.js';
+import { runToolCallChecks } from '#src/toolChecks.js';
+/** A block applies to an identity when it names no identities (unscoped → all) or explicitly names
+ * this one. For the single-run pseudo-identity (`undefined`) only unscoped blocks apply — and the
+ * parser guarantees a no-identities suite has only unscoped blocks. */
+function expectationAppliesTo(expectation, identity) {
+    if (!expectation.identities || expectation.identities.length === 0)
+        return true;
+    return identity !== undefined && expectation.identities.includes(identity);
+}
 /**
- * Build one {@link MatrixCell} per case, `content` = the case's `prompt`. Unlike BATCH-1's
- * `buildMatrix` (a models × rows cross product), eval has exactly one cell per case — so cells are
- * built directly rather than via `buildMatrix`/`bindCellContent`, which would force an artificial
- * single-row/single-model matrix per case for no benefit (no `{{field}}` interpolation applies to
- * an eval case's literal `prompt`).
- *
- * `cell.id` is the case's own `id` (not a `cell-<m>-<r>` index) so per-case JSON output filenames
- * read naturally (`<case-id>.json` — see `#src/evalOutput.js`) and results are trivially traceable
- * back to the authored case. `parseEvalSuite` already rejects duplicate case ids, so this can't
- * collide.
+ * Reduce the suite to its (case × identity) units. A suite with no `identities` reduces to a single
+ * pseudo-identity (`undefined`) so the runner has ONE code path — the whole point of the BATCH-12
+ * design: flat cases are just the one-identity, one-unscoped-block special case of the matrix.
  */
-function buildEvalCells(cases) {
-    return cases.map((evalCase, index) => ({
-        id: evalCase.id,
-        modelIndex: 0,
-        inputIndex: index,
-        content: evalCase.prompt,
-    }));
+function buildUnits(suite) {
+    const identities = suite.identities && suite.identities.length > 0 ? suite.identities : [undefined];
+    const units = [];
+    for (const evalCase of suite.cases) {
+        for (const identity of identities) {
+            units.push({
+                cellId: identity === undefined ? evalCase.id : `${evalCase.id}__${identity}`,
+                evalCase,
+                identity,
+                // BATCH-12 Task 2: filter EACH turn's blocks for this identity (length 1 = single-turn).
+                applicablePerTurn: evalCase.turns.map((turn) => turn.expectations.filter((e) => expectationAppliesTo(e, identity))),
+            });
+        }
+    }
+    return units;
 }
 /**
- * Run every case in the suite through the SUT ({@link RunEvalSuiteOptions.runCell}, pooled via
- * BATCH-1's `runBatchMatrix`), then grade each answer with deterministic checks and (when the case
- * declares a rubric) the judge. A case PASSES iff its deterministic checks pass AND (it has no
- * judge rubric OR the judge's rate is at/above the case's `passThreshold`) — ported from the field
- * user's proven harness semantics (`docs/batch-eval-user-requirements.md` Appendix A).
+ * Run every (case × identity) cell of the suite through the SUT ({@link RunCellFn}, pooled via
+ * BATCH-1's `runBatchMatrix`), then grade each answer with the applicable expectation blocks'
+ * deterministic checks, tool-trace checks, and (when a block declares a rubric) the judge. A cell
+ * PASSES iff EVERY applicable block passes — ported from the field user's proven harness semantics
+ * (`docs/batch-eval-user-requirements.md` Appendix A), generalized to per-identity blocks.
+ *
+ * ONE concurrency pool: flat and matrix suites both normalize to {@link EvalUnit}s and ride the same
+ * `runBatchMatrix` pool; the only divergence is which `RunCellFn` a unit's identity resolves to.
+ *
+ * BATCH-12 Task 2 — a unit whose case has `turns.length > 1` is a MULTI-TURN conversation: it runs
+ * through the injected {@link RunConversationFn} seam (agent/tools built once, messages accumulated
+ * across turns) instead of {@link RunCellFn}, still inside the SAME pool, and is graded turn-by-turn
+ * by {@link gradeConversationUnit} — the cell PASSES iff EVERY turn's applicable blocks pass. A
+ * single-turn unit keeps the proven `runCell` + {@link gradeUnit} path byte-for-byte.
  */
 export async function runEvalSuite(suite, options) {
-    const cells = buildEvalCells(suite.cases);
+    const units = buildUnits(suite);
+    // I1 — dispatch AND grading are keyed by the guaranteed-unique `inputIndex` (a unit's position in
+    // this array, threaded onto its `MatrixCell`/`CellResult`), NEVER by the composite `cellId`
+    // (`<caseId>__<identity>`). Both case ids and identity names permit `__`, so distinct cells can
+    // collapse to the same `cellId` (e.g. case `x` + identity `y__z` and case `x__y` + identity `z`
+    // both → `x__y__z`). A `cellId`-keyed Map would silently run/grade one cell under the WRONG
+    // identity and drop the other — the worst failure for an authorization matrix. `cellId` is kept
+    // for display and per-cell output filenames only (`evalOutput.ts` guards the filename collision).
+    const unitByInputIndex = new Map(units.map((unit, index) => [index, unit]));
+    // Resolve the RunCellFn for a unit's identity: the single `runCell` for the no-identities path,
+    // else the per-identity runCell the command built. A missing runCell is a wiring bug — surface it
+    // as a thrown error (caught by `runBatchMatrix` as a failed cell) rather than a silent skip.
+    const runCellFor = (identity) => {
+        if (identity === undefined) {
+            if (!options.runCell) {
+                throw new Error('runEvalSuite: no `runCell` provided for the single-run (no-identities) path.');
+            }
+            return options.runCell;
+        }
+        const identityRunCell = options.runCellByIdentity?.get(identity);
+        if (!identityRunCell) {
+            throw new Error(`runEvalSuite: no runCell provided for identity "${identity}".`);
+        }
+        return identityRunCell;
+    };
+    // BATCH-12 Task 2 — the MULTI-TURN counterpart to `runCellFor`: resolve the conversational seam
+    // for a unit's identity. Same missing-seam-is-a-wiring-bug discipline (throw → caught by
+    // `runBatchMatrix` as a failed cell → the whole conversation FAILs) rather than a silent skip.
+    const runConversationFor = (identity) => {
+        if (identity === undefined) {
+            if (!options.runConversation) {
+                throw new Error('runEvalSuite: no `runConversation` provided for the multi-turn (no-identities) path.');
+            }
+            return options.runConversation;
+        }
+        const identityRunConversation = options.runConversationByIdentity?.get(identity);
+        if (!identityRunConversation) {
+            throw new Error(`runEvalSuite: no runConversation provided for identity "${identity}".`);
+        }
+        return identityRunConversation;
+    };
+    const cells = units.map((unit, index) => ({
+        id: unit.cellId,
+        modelIndex: 0,
+        inputIndex: index,
+        // Single-turn dispatch reads `content`; the multi-turn branch reads the unit's turns directly
+        // (a MatrixCell has one `content`, so it can't carry a conversation) — turns[0].user is a
+        // harmless placeholder there.
+        content: unit.evalCase.turns[0].user,
+    }));
+    // BATCH-12 Task 2 — per-conversation per-turn outcomes, keyed by the unique `inputIndex` (a
+    // side-channel so a whole conversation still rides ONE `runBatchMatrix` pool with the single-turn
+    // cells — `CellRunOutcome` carries one answer, a conversation carries N). Only multi-turn cells
+    // populate this; single-turn cells grade straight from their `CellResult` as before.
+    const conversationOutcomes = new Map();
+    const dispatchRunCell = async (cell) => {
+        const unit = unitByInputIndex.get(cell.inputIndex);
+        // Single-turn: the proven `runSingleShot`-backed path, byte-for-byte (unchanged dispatch).
+        if (unit.evalCase.turns.length <= 1) {
+            return runCellFor(unit.identity)(cell);
+        }
+        // Multi-turn: run the whole conversation ONCE (agent/tools built once), stash the per-turn
+        // outcomes for grading, and report a synthetic pool outcome (`ok` iff every turn ran) so the
+        // pool's own bookkeeping is coherent — grading reads the stash, not this outcome's answer.
+        const outcomes = await runConversationFor(unit.identity)(unit.evalCase.turns.map((turn) => turn.user));
+        conversationOutcomes.set(cell.inputIndex, outcomes);
+        return { ok: outcomes.length > 0 && outcomes.every((o) => o.ok) };
+    };
     const cellResults = await runBatchMatrix(cells, {
-        runCell: options.runCell,
+        runCell: dispatchRunCell,
         concurrency: options.concurrency,
     });
-    const casesById = new Map(suite.cases.map((evalCase) => [evalCase.id, evalCase]));
     const results = [];
     for (const cellResult of cellResults) {
-        const evalCase = casesById.get(cellResult.id);
-        /* istanbul ignore next -- cell ids are derived 1:1 from case ids by buildEvalCells above */
-        if (!evalCase)
+        const unit = unitByInputIndex.get(cellResult.inputIndex);
+        /* istanbul ignore next -- inputIndex is derived 1:1 from units (each cell's inputIndex is its
+           unit's array position, preserved through runBatchMatrix), so every result maps to one unit */
+        if (!unit)
             continue;
-        results.push(await gradeCase(evalCase, cellResult, options.judge));
+        if (unit.evalCase.turns.length <= 1) {
+            results.push(await gradeUnit(unit, cellResult, options.judge));
+        }
+        else {
+            results.push(await gradeConversationUnit(unit, conversationOutcomes.get(cellResult.inputIndex), cellResult, options.judge));
+        }
     }
     const passed = results.filter((result) => result.verdict === 'PASS').length;
     return { total: results.length, passed, failed: results.length - passed, cases: results };
 }
-async function gradeCase(evalCase, cellResult, judge) {
+/**
+ * BATCH-11 (#405 his #6) — classify a completed suite's {@link EvalSuiteSummary} into a distinct
+ * exit code so CI can tell a product regression from a broken harness. BATCH-12 counts matrix
+ * CELLS (one per case × identity), not cases:
+ *
+ * - `0` — every cell passed (unchanged contract).
+ * - `1` — the suite **ran** but ≥1 cell FAILED (assertion/judge below threshold). A *product*
+ *   signal: real, gradeable results, some below the bar.
+ * - `2` — **harness error**: no gradeable results at all — an empty suite, or **every** cell's SUT
+ *   run failed (`sutOk === false`). (The other harness errors that never reach a summary — suite
+ *   load/parse error, config error, an unresolved identity precondition — are mapped to `2` by the
+ *   caller (`evalCommand.ts`) in a try/catch.)
+ *
+ * Classification is anchored on `sutOk`, not the verdict: a cell that ran (`sutOk === true`) but
+ * whose judge errored (or whose answer failed a check) is a real result → exit `1`, never `2`. A
+ * *mix* of `sutOk:false` and `sutOk:true` cells therefore yields `1`.
+ */
+export function classifyEvalExit(summary) {
+    // No gradeable results at all → harness/environment error, not a product signal.
+    if (summary.total === 0 || summary.cases.every((result) => !result.sutOk)) {
+        return 2;
+    }
+    // Ran and produced gradeable results, but at least one cell failed → product regression.
+    if (summary.failed > 0) {
+        return 1;
+    }
+    // Every cell passed.
+    return 0;
+}
+/**
+ * Grade ONE answer (+ its tool trace) against a set of APPLICABLE expectation blocks — the shared
+ * inner loop of both the single-turn {@link gradeUnit} and the multi-turn {@link gradeConversationUnit}.
+ * Runs each block's deterministic checks, tool-trace checks, and judge (when it declares a rubric),
+ * accumulating failure `reasons` (a passing block/judge appends nothing). `judgeOutcome` is the FIRST
+ * block that declared a judge, so a single-block case's `judge` field is exactly that block's outcome
+ * (back-compat). For a single applicable block this reproduces the pre-BATCH-12 grading byte-for-byte.
+ */
+async function gradeApplicableBlocks(answer, tools, applicable, passThreshold, judge) {
+    const deterministicFailures = [];
+    const reasons = [];
+    let judgeOutcome;
+    for (const block of applicable) {
+        const checks = runDeterministicChecks(answer, block);
+        // Tool-trace assertions read the captured tool names, not the answer, so they are graded
+        // separately (see #src/toolChecks.js) and merged into `reasons` alongside the answer checks.
+        const toolFailures = runToolCallChecks(tools, block);
+        deterministicFailures.push(...checks.failures);
+        reasons.push(...checks.failures, ...toolFailures);
+        if (block.judgeRubric) {
+            let outcome;
+            if (!judge) {
+                outcome = { attempted: false, ok: false, error: 'No judge configured for this suite.' };
+                reasons.push('judge: not configured');
+            }
+            else {
+                outcome = await judge(answer, block.judgeRubric);
+                if (!outcome.ok || !outcome.verdict) {
+                    reasons.push(`judge error: ${outcome.error ?? 'unknown error'}`);
+                }
+                else if (outcome.verdict.rate < passThreshold) {
+                    reasons.push(`judge rate ${outcome.verdict.rate}/10 below threshold ${passThreshold}` +
+                        (outcome.verdict.reason ? `: ${outcome.verdict.reason}` : ''));
+                }
+            }
+            if (judgeOutcome === undefined)
+                judgeOutcome = outcome;
+        }
+    }
+    return { reasons, deterministicFailures, judgeOutcome };
+}
+/**
+ * Grade one SINGLE-TURN (case × identity) cell: run its (single turn's) APPLICABLE expectation
+ * blocks' deterministic checks, tool-trace checks, and judge; the cell PASSES iff every applicable
+ * block passes. For a flat case (exactly one unscoped applicable block) this produces byte-for-byte
+ * the same `reasons`/`checks`/`judge`/`verdict` as the pre-BATCH-12 single-block grader.
+ */
+async function gradeUnit(unit, cellResult, judge) {
+    const { evalCase, identity, applicablePerTurn } = unit;
+    const applicable = applicablePerTurn[0];
     const base = {
         id: evalCase.id,
+        // Omit `identity` entirely (rather than set it to `undefined`) for the no-identities path, so
+        // its `<id>.json` output stays byte-for-byte identical to before BATCH-12.
+        ...(identity !== undefined ? { identity } : {}),
         passThreshold: evalCase.passThreshold,
         answer: cellResult.answer,
         tokensInput: cellResult.tokensInput,
@@ -63,38 +235,120 @@ async function gradeCase(evalCase, cellResult, judge) {
             reasons: [cellResult.error ? `SUT run failed: ${cellResult.error}` : 'SUT run failed.'],
         };
     }
-    const answer = cellResult.answer ?? '';
-    const checks = runDeterministicChecks(answer, evalCase);
-    const reasons = [...checks.failures];
-    let judgeOutcome;
-    let judgePassed = true;
-    if (evalCase.judgeRubric) {
-        if (!judge) {
-            judgeOutcome = { attempted: false, ok: false, error: 'No judge configured for this suite.' };
-            judgePassed = false;
-            reasons.push('judge: not configured');
-        }
-        else {
-            judgeOutcome = await judge(answer, evalCase.judgeRubric);
-            if (!judgeOutcome.ok || !judgeOutcome.verdict) {
-                judgePassed = false;
-                reasons.push(`judge error: ${judgeOutcome.error ?? 'unknown error'}`);
-            }
-            else if (judgeOutcome.verdict.rate < evalCase.passThreshold) {
-                judgePassed = false;
-                reasons.push(`judge rate ${judgeOutcome.verdict.rate}/10 below threshold ${evalCase.passThreshold}` +
-                    (judgeOutcome.verdict.reason ? `: ${judgeOutcome.verdict.reason}` : ''));
-            }
-        }
+    // NO-SILENT-PASS backstop (M2): a cell with zero applicable blocks is a suite-AUTHORING error, not
+    // a product regression — nothing would grade this (case × identity). The parser rejects this
+    // statically when the suite declares identities, so this branch is only reachable if that guard is
+    // bypassed; when it is, THROW so the eval command's catch classifies it as a harness error
+    // (exit 2), never a product FAIL (exit 1) or, worse, a silent trivial pass. Throwing aborts the
+    // whole suite, which is acceptable precisely because this is unreachable under the parse-time guard.
+    if (applicable.length === 0) {
+        throw new Error(`no applicable expectation block for cell "${unit.cellId}" (identity ` +
+            `"${identity ?? '(default)'}") — nothing would grade this (case × identity), which is a ` +
+            'suite-authoring error.');
     }
-    const verdict = checks.passed && judgePassed ? 'PASS' : 'FAIL';
+    const answer = cellResult.answer ?? '';
+    const tools = cellResult.tools ?? [];
+    const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, applicable, evalCase.passThreshold, judge);
+    // A cell PASSES iff nothing was recorded against it: every applicable block's deterministic
+    // checks, tool-trace checks, and judge all passed (a passing judge appends no reason). For a
+    // single applicable block this is exactly the pre-BATCH-12 verdict.
+    const verdict = reasons.length === 0 ? 'PASS' : 'FAIL';
     return {
         ...base,
         verdict,
         sutOk: true,
-        checks,
+        checks: { passed: deterministicFailures.length === 0, failures: deterministicFailures },
         judge: judgeOutcome,
         reasons,
     };
 }
+/**
+ * Grade one MULTI-TURN (case × identity) cell (BATCH-12 Task 2): grade EACH turn's answer + per-turn
+ * tool trace against THAT turn's applicable blocks, then roll them up. The cell PASSES iff EVERY turn
+ * PASSes; its top-level `reasons` are every turn's reasons each prefixed with the failing turn
+ * (`turn N: …`) so the summary/output pinpoints which turn broke, and the full per-turn breakdown is
+ * in `turns`.
+ *
+ * `outcomes` is what the conversational runner returned (one per turn it attempted). A SHORT array
+ * (the conversation aborted mid-way) or a MISSING one (`undefined` — the runner threw, so
+ * `runBatchMatrix` recorded a failed cell with no stash) fails the un-run turns with a clear reason.
+ * `sutOk` is TRUE iff at least one turn actually ran — so a totally-failed conversation is
+ * `sutOk:false` (exit-2-eligible, like a single-shot SUT failure) while a turn-1-ran/turn-2-failed
+ * cell is `sutOk:true`, a real product signal → exit 1.
+ */
+async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
+    const { evalCase, identity, applicablePerTurn } = unit;
+    const turnOutcomes = outcomes ?? [];
+    const turnResults = [];
+    const cellReasons = [];
+    let anyTurnRan = false;
+    for (let i = 0; i < evalCase.turns.length; i++) {
+        const turn = evalCase.turns[i];
+        const applicable = applicablePerTurn[i];
+        // NO-SILENT-PASS backstop (per turn × identity), mirroring gradeUnit's: unreachable under the
+        // parse-time guard, but if bypassed THROW so it's a harness error (exit 2), never a silent pass.
+        if (applicable.length === 0) {
+            throw new Error(`no applicable expectation block for cell "${unit.cellId}" (identity ` +
+                `"${identity ?? '(default)'}") turn ${i} — nothing would grade this (turn × identity), ` +
+                'which is a suite-authoring error.');
+        }
+        const outcome = turnOutcomes[i];
+        const label = `turn ${i + 1}`;
+        if (!outcome || !outcome.ok) {
+            // This turn produced no answer: the conversation errored on it, or aborted before reaching it.
+            let detail;
+            if (outcome?.error)
+                detail = `SUT run failed: ${outcome.error}`;
+            else if (outcome)
+                detail = 'SUT run failed.';
+            else if (outcomes === undefined)
+                detail = `SUT run failed: ${cellResult.error ?? 'the conversation could not run.'}`;
+            else
+                detail = 'SUT run failed: the conversation ended before this turn.';
+            turnResults.push({
+                user: turn.user,
+                answer: outcome?.answer,
+                tokensInput: outcome?.tokensInput,
+                tokensOutput: outcome?.tokensOutput,
+                tools: outcome?.tools,
+                ok: false,
+                verdict: 'FAIL',
+                reasons: [detail],
+            });
+            cellReasons.push(`${label}: ${detail}`);
+            continue;
+        }
+        anyTurnRan = true;
+        const answer = outcome.answer ?? '';
+        const tools = outcome.tools ?? [];
+        const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, applicable, evalCase.passThreshold, judge);
+        const turnVerdict = reasons.length === 0 ? 'PASS' : 'FAIL';
+        turnResults.push({
+            user: turn.user,
+            answer: outcome.answer,
+            tokensInput: outcome.tokensInput,
+            tokensOutput: outcome.tokensOutput,
+            tools: outcome.tools,
+            ok: true,
+            verdict: turnVerdict,
+            checks: { passed: deterministicFailures.length === 0, failures: deterministicFailures },
+            judge: judgeOutcome,
+            reasons,
+        });
+        for (const reason of reasons)
+            cellReasons.push(`${label}: ${reason}`);
+    }
+    const verdict = cellReasons.length === 0 ? 'PASS' : 'FAIL';
+    return {
+        id: evalCase.id,
+        ...(identity !== undefined ? { identity } : {}),
+        passThreshold: evalCase.passThreshold,
+        // A multi-turn cell has no single answer/tools/checks/judge — read the per-turn breakdown below.
+        durationMs: cellResult.durationMs,
+        verdict,
+        sutOk: anyTurnRan,
+        reasons: cellReasons,
+        turns: turnResults,
+    };
+}
 //# sourceMappingURL=evalRunner.js.map

package/dist/evalRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AA2BrE;;;;;;;;;;;GAWG;AACH,SAAS,cAAc,CAAC,KAAiB;IACvC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACrC,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,OAAO,EAAE,QAAQ,CAAC,MAAM;KACzB,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC9C,4FAA4F;QAC5F,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAkB,EAClB,UAAsB,EACtB,KAA0B;IAE1B,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAErC,IAAI,YAAsC,CAAC;IAC3C,IAAI,WAAW,GAAG,IAAI,CAAC;IACvB,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,YAAY,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;YAC7F,WAAW,GAAG,KAAK,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YACzD,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBAC9C,WAAW,GAAG,KAAK,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;YACxE,CAAC;iBAAM,IAAI,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAC9D,WAAW,GAAG,KAAK,CAAC;gBACpB,OAAO,CAAC,IAAI,CACV,cAAc,YAAY,CAAC,OAAO,CAAC,IAAI,uBAAuB,QAAQ,CAAC,aAAa,EAAE;oBACpF,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAoB,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEhF,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM;QACN,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAwEvD;;uEAEuE;AACvE,SAAS,oBAAoB,CAAC,WAA4B,EAAE,QAA4B;IACtF,IAAI,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,OAAO,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE;gBAC5E,QAAQ;gBACR,QAAQ;gBACR,yFAAyF;gBACzF,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CACnE;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,kGAAkG;IAClG,4FAA4F;IAC5F,gGAAgG;IAChG,gGAAgG;IAChG,4FAA4F;IAC5F,gGAAgG;IAChG,kGAAkG;IAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5E,gGAAgG;IAChG,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,UAAU,GAAG,CAAC,QAA4B,EAAa,EAAE;QAC7D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,gGAAgG;IAChG,yFAAyF;IACzF,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,CAAC,QAA4B,EAAqB,EAAE;QAC7E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,eAAe,CAAC;QACjC,CAAC;QACD,MAAM,uBAAuB,GAAG,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2DAA2D,QAAQ,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,8FAA8F;QAC9F,0FAA0F;QAC1F,8BAA8B;QAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,4FAA4F;IAC5F,kGAAkG;IAClG,gGAAgG;IAChG,qFAAqF;IACrF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,MAAM,eAAe,GAAc,KAAK,EAAE,IAAI,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;QACpD,2FAA2F;QAC3F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,2FAA2F;QAC3F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7C,CAAC;QACF,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACzD;wGACgG;QAChG,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,MAAM,qBAAqB,CACzB,IAAI,EACJ,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAC/C,UAAU,EACV,OAAO,CAAC,KAAK,CACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAKD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,iFAAiF;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yFAAyF;IACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qBAAqB;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAe,EACf,UAA6B,EAC7B,aAAqB,EACrB,KAA0B;IAM1B,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,qBAAqB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,CAAC;QAElD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,OAAqB,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACjD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;oBAChD,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,OAAO,CAAC,IAAI,uBAAuB,aAAa,EAAE;wBACtE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,GAAG,OAAO,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,8FAA8F;QAC9F,2EAA2E;QAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,6FAA6F;IAC7F,mGAAmG;IACnG,2FAA2F;IAC3F,gGAAgG;IAChG,qGAAqG;IACrG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;YACpE,IAAI,QAAQ,IAAI,WAAW,8DAA8D;YACzF,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,OAAO,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,qBAAqB,CAClC,IAAc,EACd,QAAsC,EACtC,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,IAAI,EAAE,CAAC;IAEpC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAExC,8FAA8F;QAC9F,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;gBACpE,IAAI,QAAQ,IAAI,WAAW,WAAW,CAAC,iDAAiD;gBACxF,mCAAmC,CACtC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAC5B,+FAA+F;YAC/F,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,EAAE,KAAK;gBAAE,MAAM,GAAG,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC;iBAC3D,IAAI,OAAO;gBAAE,MAAM,GAAG,iBAAiB,CAAC;iBACxC,IAAI,QAAQ,KAAK,SAAS;gBAC7B,MAAM,GAAG,mBAAmB,UAAU,CAAC,KAAK,IAAI,iCAAiC,EAAE,CAAC;;gBACjF,MAAM,GAAG,0DAA0D,CAAC;YAEzE,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,YAAY,EAAE,OAAO,EAAE,YAAY;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK;gBACrB,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;QACF,MAAM,WAAW,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;YACvF,KAAK,EAAE,YAAY;YACnB,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iGAAiG;QACjG,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO;QACP,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC"}

package/dist/evalSuite.d.ts

@@ -2,21 +2,42 @@ import type { EvalSuite } from '#src/evalTypes.js';
 /**
  * Parse and validate an eval suite YAML document into a normalized {@link EvalSuite}.
  *
+ * Every case is normalized to ONE shape — `turns: [{ user, expectations: EvalExpectation[] }]` (Task
+ * 1: exactly one turn). A flat case is sugar for one unscoped expectation (applies to every
+ * identity); a matrix case's `expect:` array becomes the expectation list directly.
+ *
  * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
  * - Malformed YAML.
  * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
- * - `target.type` other than `"gth-agent"` — pluggable CLI/HTTP targets are out of scope for this
- *   task (see the CLI surface doc's future `target: {type: cli|http}` sketch).
+ * - `target.type` other than `"gth-agent"`, `"adk-agent"`, or `"ag-ui"` — other pluggable CLI/HTTP
+ *   targets are out of scope.
  * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
- *   switch is the same `--identities` direction the brief scopes out; this task's target is always
- *   whatever profile `gth eval` itself was invoked under (`-i/--identity-profile`, if any).
- * - A case `id` containing anything other than alphanumerics, dashes, underscores, or dots (case
- *   ids double as output filenames — see `#src/evalOutput.js` — so a path separator or traversal
- *   sequence like `../../etc/passwd` is rejected here, not sanitized).
- * - A duplicate case `id` (case ids double as output filenames — see `#src/evalOutput.js`).
- * - A case with **no** deterministic checks (`must_contain`/`must_not_contain`/
- *   `should_contain_any` all absent/empty) **and no** `judge` rubric: nothing would ever grade it,
- *   which is a suite-authoring bug, not a case that trivially passes.
+ *   switch is the `--identities` direction, replaced by the suite-level `identities` list.
+ * - A `"adk-agent"` (BATCH-14) target missing its `url`; an `adk-agent` suite that ALSO uses the
+ *   `identities` matrix (per-identity gth configs are meaningless for an external agent); or an
+ *   `adk-agent` suite that uses `must_call`/`must_not_call` (A2A does not expose the agent's tool
+ *   trace, so grading a tool-call assertion is impossible — rejected rather than silently passed).
+ * - An `"ag-ui"` (BATCH-15) target missing its `url` or its `agent_id`; an `ag-ui` target carrying a
+ *   `profile`; or an `ag-ui` suite that ALSO uses the `identities` matrix (per-identity gth configs
+ *   are meaningless for an external agent). NOTE `must_call`/`must_not_call` ARE supported for an
+ *   `ag-ui` target — the AG-UI wire streams `TOOL_CALL_START`, so the tool trace is captured and
+ *   graded (this is the key difference from `adk-agent`).
+ * - A case `id`, or a suite `identities` name, containing anything other than alphanumerics,
+ *   dashes, underscores, or dots (both double as output filenames — path traversal is rejected).
+ * - A duplicate case `id`, or a duplicate `identities` entry.
+ * - A case declaring BOTH `prompt` and `turns:`, or NEITHER (a case is single- or multi-turn).
+ * - A single-turn case with a missing/blank `prompt`; a multi-turn turn with a missing/blank `user`;
+ *   an empty `turns:` array.
+ * - A multi-turn case that ALSO declares case-level assertions or an `expect:` array (those live on
+ *   each turn for a multi-turn case).
+ * - A case (or turn) declaring BOTH flat assertions AND an `expect:` array (one way per case/turn).
+ * - An `expect:` block referencing an identity the suite does not declare.
+ * - A (turn × identity) with no applicable expectation block (statically determinable → rejected
+ *   here; the runner also guards it as a per-cell FAIL as a backstop).
+ * - A flat case or `expect:` block with no checks of any kind AND no `judge` rubric.
+ * - An invalid `must_match`/`must_not_match` regex, or a `json_path` entry not setting exactly one
+ *   of `equals`/`contains`.
+ * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.
  * @param sourcePath Optional path, only used to make error messages more actionable.