@gaunt-sloth/batch 2.0.0-alpha.21 → 2.0.0-alpha.23

package/dist/evalRunner.js

@@ -1,6 +1,7 @@
 import { runBatchMatrix } from '#src/BatchRunner.js';
 import { runDeterministicChecks } from '#src/deterministicChecks.js';
 import { runToolCallChecks } from '#src/toolChecks.js';
+import { runToolResultChecks } from '#src/toolResultChecks.js';
 /** A block applies to an identity when it names no identities (unscoped → all) or explicitly names
  * this one. For the single-run pseudo-identity (`undefined`) only unscoped blocks apply — and the
  * parser guarantees a no-identities suite has only unscoped blocks. */
@@ -168,12 +169,13 @@ export function classifyEvalExit(summary) {
 /**
  * Grade ONE answer (+ its tool trace) against a set of APPLICABLE expectation blocks — the shared
  * inner loop of both the single-turn {@link gradeUnit} and the multi-turn {@link gradeConversationUnit}.
- * Runs each block's deterministic checks, tool-trace checks, and judge (when it declares a rubric),
- * accumulating failure `reasons` (a passing block/judge appends nothing). `judgeOutcome` is the FIRST
- * block that declared a judge, so a single-block case's `judge` field is exactly that block's outcome
- * (back-compat). For a single applicable block this reproduces the pre-BATCH-12 grading byte-for-byte.
+ * Runs each block's deterministic checks, tool-trace checks, tool-RESULT checks (BATCH-21), and
+ * judge (when it declares a rubric), accumulating failure `reasons` (a passing block/judge appends
+ * nothing). `judgeOutcome` is the FIRST block that declared a judge, so a single-block case's
+ * `judge` field is exactly that block's outcome (back-compat). For a single applicable block this
+ * reproduces the pre-BATCH-12 grading byte-for-byte.
  */
-async function gradeApplicableBlocks(answer, tools, applicable, passThreshold, judge) {
+async function gradeApplicableBlocks(answer, tools, toolResults, applicable, passThreshold, judge) {
     const deterministicFailures = [];
     const reasons = [];
     let judgeOutcome;
@@ -182,8 +184,12 @@ async function gradeApplicableBlocks(answer, tools, applicable, passThreshold, j
         // Tool-trace assertions read the captured tool names, not the answer, so they are graded
         // separately (see #src/toolChecks.js) and merged into `reasons` alongside the answer checks.
         const toolFailures = runToolCallChecks(tools, block);
+        // BATCH-21 tool-RESULT assertions read the captured per-call results (isError + payload) — a
+        // third input kind, graded by its own checker (#src/toolResultChecks.js) and merged into the
+        // same `reasons` so they drive the same PASS/FAIL/exit contract.
+        const toolResultFailures = runToolResultChecks(toolResults, block);
         deterministicFailures.push(...checks.failures);
-        reasons.push(...checks.failures, ...toolFailures);
+        reasons.push(...checks.failures, ...toolFailures, ...toolResultFailures);
         if (block.judgeRubric) {
             let outcome;
             if (!judge) {
@@ -225,6 +231,9 @@ async function gradeUnit(unit, cellResult, judge) {
         tokensInput: cellResult.tokensInput,
         tokensOutput: cellResult.tokensOutput,
         tools: cellResult.tools,
+        // BATCH-21: threaded parallel to `tools`; `undefined` for a runner that captured none, so a
+        // pre-BATCH-21 cell's `<id>.json` keeps its exact keys (JSON.stringify drops undefined).
+        toolResults: cellResult.toolResults,
         durationMs: cellResult.durationMs,
     };
     if (!cellResult.ok) {
@@ -248,7 +257,8 @@ async function gradeUnit(unit, cellResult, judge) {
     }
     const answer = cellResult.answer ?? '';
     const tools = cellResult.tools ?? [];
-    const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, applicable, evalCase.passThreshold, judge);
+    const toolResults = cellResult.toolResults ?? [];
+    const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge);
     // A cell PASSES iff nothing was recorded against it: every applicable block's deterministic
     // checks, tool-trace checks, and judge all passed (a passing judge appends no reason). For a
     // single applicable block this is exactly the pre-BATCH-12 verdict.
@@ -311,6 +321,7 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
                 tokensInput: outcome?.tokensInput,
                 tokensOutput: outcome?.tokensOutput,
                 tools: outcome?.tools,
+                toolResults: outcome?.toolResults,
                 ok: false,
                 verdict: 'FAIL',
                 reasons: [detail],
@@ -321,7 +332,8 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
         anyTurnRan = true;
         const answer = outcome.answer ?? '';
         const tools = outcome.tools ?? [];
-        const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, applicable, evalCase.passThreshold, judge);
+        const toolResults = outcome.toolResults ?? [];
+        const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge);
         const turnVerdict = reasons.length === 0 ? 'PASS' : 'FAIL';
         turnResults.push({
             user: turn.user,
@@ -329,6 +341,7 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
             tokensInput: outcome.tokensInput,
             tokensOutput: outcome.tokensOutput,
             tools: outcome.tools,
+            toolResults: outcome.toolResults,
             ok: true,
             verdict: turnVerdict,
             checks: { passed: deterministicFailures.length === 0, failures: deterministicFailures },

package/dist/evalRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAwEvD;;uEAEuE;AACvE,SAAS,oBAAoB,CAAC,WAA4B,EAAE,QAA4B;IACtF,IAAI,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,OAAO,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE;gBAC5E,QAAQ;gBACR,QAAQ;gBACR,yFAAyF;gBACzF,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CACnE;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,kGAAkG;IAClG,4FAA4F;IAC5F,gGAAgG;IAChG,gGAAgG;IAChG,4FAA4F;IAC5F,gGAAgG;IAChG,kGAAkG;IAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5E,gGAAgG;IAChG,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,UAAU,GAAG,CAAC,QAA4B,EAAa,EAAE;QAC7D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,gGAAgG;IAChG,yFAAyF;IACzF,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,CAAC,QAA4B,EAAqB,EAAE;QAC7E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,eAAe,CAAC;QACjC,CAAC;QACD,MAAM,uBAAuB,GAAG,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2DAA2D,QAAQ,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,8FAA8F;QAC9F,0FAA0F;QAC1F,8BAA8B;QAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,4FAA4F;IAC5F,kGAAkG;IAClG,gGAAgG;IAChG,qFAAqF;IACrF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,MAAM,eAAe,GAAc,KAAK,EAAE,IAAI,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;QACpD,2FAA2F;QAC3F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,2FAA2F;QAC3F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7C,CAAC;QACF,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACzD;wGACgG;QAChG,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,MAAM,qBAAqB,CACzB,IAAI,EACJ,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAC/C,UAAU,EACV,OAAO,CAAC,KAAK,CACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAKD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,iFAAiF;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yFAAyF;IACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qBAAqB;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAe,EACf,UAA6B,EAC7B,aAAqB,EACrB,KAA0B;IAM1B,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,qBAAqB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,CAAC;QAElD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,OAAqB,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACjD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;oBAChD,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,OAAO,CAAC,IAAI,uBAAuB,aAAa,EAAE;wBACtE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,GAAG,OAAO,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,8FAA8F;QAC9F,2EAA2E;QAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,6FAA6F;IAC7F,mGAAmG;IACnG,2FAA2F;IAC3F,gGAAgG;IAChG,qGAAqG;IACrG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;YACpE,IAAI,QAAQ,IAAI,WAAW,8DAA8D;YACzF,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,OAAO,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,qBAAqB,CAClC,IAAc,EACd,QAAsC,EACtC,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,IAAI,EAAE,CAAC;IAEpC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAExC,8FAA8F;QAC9F,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;gBACpE,IAAI,QAAQ,IAAI,WAAW,WAAW,CAAC,iDAAiD;gBACxF,mCAAmC,CACtC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAC5B,+FAA+F;YAC/F,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,EAAE,KAAK;gBAAE,MAAM,GAAG,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC;iBAC3D,IAAI,OAAO;gBAAE,MAAM,GAAG,iBAAiB,CAAC;iBACxC,IAAI,QAAQ,KAAK,SAAS;gBAC7B,MAAM,GAAG,mBAAmB,UAAU,CAAC,KAAK,IAAI,iCAAiC,EAAE,CAAC;;gBACjF,MAAM,GAAG,0DAA0D,CAAC;YAEzE,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,YAAY,EAAE,OAAO,EAAE,YAAY;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK;gBACrB,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;QACF,MAAM,WAAW,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;YACvF,KAAK,EAAE,YAAY;YACnB,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iGAAiG;QACjG,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO;QACP,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAwE/D;;uEAEuE;AACvE,SAAS,oBAAoB,CAAC,WAA4B,EAAE,QAA4B;IACtF,IAAI,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,OAAO,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE;gBAC5E,QAAQ;gBACR,QAAQ;gBACR,yFAAyF;gBACzF,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CACnE;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,kGAAkG;IAClG,4FAA4F;IAC5F,gGAAgG;IAChG,gGAAgG;IAChG,4FAA4F;IAC5F,gGAAgG;IAChG,kGAAkG;IAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5E,gGAAgG;IAChG,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,UAAU,GAAG,CAAC,QAA4B,EAAa,EAAE;QAC7D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,gGAAgG;IAChG,yFAAyF;IACzF,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,CAAC,QAA4B,EAAqB,EAAE;QAC7E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,eAAe,CAAC;QACjC,CAAC;QACD,MAAM,uBAAuB,GAAG,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2DAA2D,QAAQ,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,8FAA8F;QAC9F,0FAA0F;QAC1F,8BAA8B;QAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,4FAA4F;IAC5F,kGAAkG;IAClG,gGAAgG;IAChG,qFAAqF;IACrF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,MAAM,eAAe,GAAc,KAAK,EAAE,IAAI,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;QACpD,2FAA2F;QAC3F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,2FAA2F;QAC3F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7C,CAAC;QACF,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACzD;wGACgG;QAChG,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,MAAM,qBAAqB,CACzB,IAAI,EACJ,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAC/C,UAAU,EACV,OAAO,CAAC,KAAK,CACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAKD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,iFAAiF;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yFAAyF;IACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qBAAqB;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAe,EACf,WAA+B,EAC/B,UAA6B,EAC7B,aAAqB,EACrB,KAA0B;IAM1B,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,6FAA6F;QAC7F,6FAA6F;QAC7F,iEAAiE;QACjE,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE,qBAAqB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,GAAG,kBAAkB,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,OAAqB,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACjD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;oBAChD,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,OAAO,CAAC,IAAI,uBAAuB,aAAa,EAAE;wBACtE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,GAAG,OAAO,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,8FAA8F;QAC9F,2EAA2E;QAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,4FAA4F;QAC5F,yFAAyF;QACzF,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,6FAA6F;IAC7F,mGAAmG;IACnG,2FAA2F;IAC3F,gGAAgG;IAChG,qGAAqG;IACrG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;YACpE,IAAI,QAAQ,IAAI,WAAW,8DAA8D;YACzF,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,EAAE,CAAC;IACjD,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,OAAO,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,qBAAqB,CAClC,IAAc,EACd,QAAsC,EACtC,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,IAAI,EAAE,CAAC;IAEpC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAExC,8FAA8F;QAC9F,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;gBACpE,IAAI,QAAQ,IAAI,WAAW,WAAW,CAAC,iDAAiD;gBACxF,mCAAmC,CACtC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAC5B,+FAA+F;YAC/F,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,EAAE,KAAK;gBAAE,MAAM,GAAG,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC;iBAC3D,IAAI,OAAO;gBAAE,MAAM,GAAG,iBAAiB,CAAC;iBACxC,IAAI,QAAQ,KAAK,SAAS;gBAC7B,MAAM,GAAG,mBAAmB,UAAU,CAAC,KAAK,IAAI,iCAAiC,EAAE,CAAC;;gBACjF,MAAM,GAAG,0DAA0D,CAAC;YAEzE,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,YAAY,EAAE,OAAO,EAAE,YAAY;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK;gBACrB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;QAC9C,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;QACF,MAAM,WAAW,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;YACvF,KAAK,EAAE,YAAY;YACnB,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iGAAiG;QACjG,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO;QACP,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC"}

package/dist/evalSuite.d.ts

@@ -36,7 +36,11 @@ import type { EvalSuite } from '#src/evalTypes.js';
  *   here; the runner also guards it as a per-cell FAIL as a backstop).
  * - A flat case or `expect:` block with no checks of any kind AND no `judge` rubric.
  * - An invalid `must_match`/`must_not_match` regex, or a `json_path` entry not setting exactly one
- *   of `equals`/`contains`.
+ *   of `equals`/`contains`, or a `tool_result_json_path` entry setting BOTH (at most one; neither
+ *   is a pure existence check).
+ * - A tool-RESULT assertion (`must_error` / `tool_result_json_path`, BATCH-21) against an
+ *   `adk-agent` OR `ag-ui` target — tool results exist only on the in-process `gth-agent` target
+ *   (the AG-UI wire streams call names but no result payloads; A2A exposes no tool trace at all).
  * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.

package/dist/evalSuite.js

@@ -39,6 +39,9 @@ import { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
  *     json_path:                                # BATCH-10: over the answer parsed as JSON
  *       - { path: "$.items[0].scope", equals: "caller" }
  *       - { path: "data.status", contains: "ok" }
+ *     must_error: [ "mcp__unimarket__*" ]       # BATCH-21: a matching called tool RETURNED an error
+ *     tool_result_json_path:                    # BATCH-21: over a matching tool RESULT's payload
+ *       - { tool: "mcp__unimarket__*", path: "error.code", equals: "ACCESS_DENIED" }
  *     judge: "Answers with a ranked summary and correctly formatted values."
  *     pass_threshold: 7
  *   # Matrix case — per-identity expectations:
@@ -58,6 +61,15 @@ const RawJsonPathCheckSchema = z.object({
     equals: z.unknown().optional(),
     contains: z.string().optional(),
 });
+/** BATCH-21 — one `tool_result_json_path` entry: `json_path`'s shape plus the `tool` name pattern
+ * selecting whose result to check. `equals`/`contains` exclusivity (at most one; neither = pure
+ * existence check) is enforced in code, mirroring `json_path`'s. */
+const RawToolResultJsonPathCheckSchema = z.object({
+    tool: z.string().min(1, 'tool_result_json_path entry must have a non-empty tool pattern'),
+    path: z.string().min(1, 'tool_result_json_path entry must have a non-empty path'),
+    equals: z.unknown().optional(),
+    contains: z.string().optional(),
+});
 /** The assertion bundle keys shared by a flat case and an `expect:` block. `expect:` blocks may also
  * carry `identities`; the flat case has no `identities` key (it always applies to every identity). */
 const RawAssertionsSchema = z.object({
@@ -69,6 +81,9 @@ const RawAssertionsSchema = z.object({
     must_match: z.array(z.string()).optional(),
     must_not_match: z.array(z.string()).optional(),
     json_path: z.array(RawJsonPathCheckSchema).optional(),
+    // BATCH-21 tool-RESULT assertions (gth-agent target only, enforced after normalization below).
+    must_error: z.array(z.string()).optional(),
+    tool_result_json_path: z.array(RawToolResultJsonPathCheckSchema).optional(),
     judge: z.string().optional(),
 });
 const RawExpectationSchema = RawAssertionsSchema.extend({
@@ -139,6 +154,8 @@ const FLAT_ASSERTION_KEYS = [
     'must_match',
     'must_not_match',
     'json_path',
+    'must_error',
+    'tool_result_json_path',
     'judge',
 ];
 /** A plain profile-name pattern — same as a case id: alphanumerics, dashes, underscores, dots. This
@@ -182,7 +199,11 @@ const IDENTITY_NAME_RE = /^[\w.-]+$/;
  *   here; the runner also guards it as a per-cell FAIL as a backstop).
  * - A flat case or `expect:` block with no checks of any kind AND no `judge` rubric.
  * - An invalid `must_match`/`must_not_match` regex, or a `json_path` entry not setting exactly one
- *   of `equals`/`contains`.
+ *   of `equals`/`contains`, or a `tool_result_json_path` entry setting BOTH (at most one; neither
+ *   is a pure existence check).
+ * - A tool-RESULT assertion (`must_error` / `tool_result_json_path`, BATCH-21) against an
+ *   `adk-agent` OR `ag-ui` target — tool results exist only on the in-process `gth-agent` target
+ *   (the AG-UI wire streams call names but no result payloads; A2A exposes no tool trace at all).
  * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.
@@ -402,6 +423,28 @@ export function parseEvalSuite(yamlText, sourcePath) {
             }
         }
     }
+    // BATCH-21 — the honest tool-RESULT boundary, for BOTH external targets. Only the in-process
+    // `gth-agent` target produces `ToolMessage`s, so only it can carry tool RESULTS: the `adk-agent`
+    // (A2A) wire has an invisible tool trace entirely, and the `ag-ui` wire streams TOOL_CALL_START
+    // events (call NAMES — which is why `must_call` IS graded there) but no result payloads. A
+    // tool-result assertion against either would be ungradeable, and an ungradeable assertion that
+    // silently PASSES is the worst outcome for an eval tool — reject it at parse time, naming the
+    // offending case (→ the eval command's catch → exit 2), exactly like the adk `must_call` guard.
+    if (target.type === 'adk-agent' || target.type === 'ag-ui') {
+        for (const evalCase of cases) {
+            for (const turn of evalCase.turns) {
+                for (const expectation of turn.expectations) {
+                    if (expectation.mustError.length > 0 || expectation.toolResultJsonPath.length > 0) {
+                        throw new Error(`Invalid eval suite${suffix}: case "${evalCase.id}" uses \`must_error\`/` +
+                            '`tool_result_json_path` — tool-result assertions require target.type: gth-agent ' +
+                            `(only the in-process agent surfaces tool results; ${target.type === 'ag-ui'
+                                ? 'the AG-UI wire streams tool-call names only, with no result payload'
+                                : "A2A does not expose the agent's tool trace at all"}). Use content assertions, or a \`gth-agent\` target for tool-result grading.`);
+                    }
+                }
+            }
+        }
+    }
     // Normalize a blank/whitespace-only judge_profile to undefined (= no separate judge) so the CLI's
     // resolution treats it the same as absent.
     const judgeProfile = data.judge_profile?.trim() || undefined;
@@ -543,6 +586,25 @@ function buildExpectation(raw, blockIdentities, ctx) {
             ? { path: entry.path, contains: entry.contains }
             : { path: entry.path, equals: entry.equals };
     });
+    // BATCH-21 tool-RESULT assertions. `must_error` needs no per-entry validation (plain name
+    // patterns, same as must_call). A tool_result_json_path entry may set AT MOST one of
+    // equals/contains — unlike json_path's exactly-one, NEITHER is valid here and means a pure
+    // path-existence check (the key is normalized OUT so graders can discriminate on key presence;
+    // an explicit `equals: null` keeps its key and asserts the value is null).
+    const mustError = raw.must_error ?? [];
+    const toolResultJsonPath = (raw.tool_result_json_path ?? []).map((entry) => {
+        const hasEquals = entry.equals !== undefined;
+        const hasContains = entry.contains !== undefined;
+        if (hasEquals && hasContains) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} tool_result_json_path entry for ` +
+                `"${entry.path}" must set at most one of "equals" or "contains" (neither = existence check).`);
+        }
+        if (hasContains)
+            return { tool: entry.tool, path: entry.path, contains: entry.contains };
+        if (hasEquals)
+            return { tool: entry.tool, path: entry.path, equals: entry.equals };
+        return { tool: entry.tool, path: entry.path };
+    });
     const hasChecks = mustContain.length > 0 ||
         mustNotContain.length > 0 ||
         shouldContainAny.length > 0 ||
@@ -550,13 +612,16 @@ function buildExpectation(raw, blockIdentities, ctx) {
         mustNotCall.length > 0 ||
         mustMatch.length > 0 ||
         mustNotMatch.length > 0 ||
-        jsonPath.length > 0;
+        jsonPath.length > 0 ||
+        mustError.length > 0 ||
+        toolResultJsonPath.length > 0;
     const judgeRubric = raw.judge?.trim();
     const hasJudge = !!judgeRubric;
     if (!hasChecks && !hasJudge) {
         throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has no checks and no judge rubric — it must ` +
             'declare at least one of must_contain / must_not_contain / should_contain_any / must_call ' +
-            '/ must_not_call / must_match / must_not_match / json_path, or a judge rubric.');
+            '/ must_not_call / must_match / must_not_match / json_path / must_error / ' +
+            'tool_result_json_path, or a judge rubric.');
     }
     return {
         identities,
@@ -568,6 +633,8 @@ function buildExpectation(raw, blockIdentities, ctx) {
         mustMatch,
         mustNotMatch,
         jsonPath,
+        mustError,
+        toolResultJsonPath,
         judgeRubric: hasJudge ? judgeRubric : undefined,
     };
 }

package/dist/evalSuite.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalSuite.js","sourceRoot":"","sources":["../src/evalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4CAA4C,CAAC;IACrE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH;sGACsG;AACtG,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC5C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9C,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,MAAM,CAAC;IACtD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH;;;mEAGmE;AACnE,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAC5C,KAAK,CACJ,WAAW,EACX,wFAAwF;QACtF,0FAA0F,CAC7F;IACH,uFAAuF;IACvF,8EAA8E;IAC9E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,iGAAiG;IACjG,mEAAmE;IACnE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;IAChD,mGAAmG;IACnG,+FAA+F;IAC/F,sDAAsD;IACtD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IACxC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC9B,0FAA0F;QAC1F,2FAA2F;QAC3F,mEAAmE;QACnE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC1B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAChC,CAAC;IACF,kGAAkG;IAClG,gGAAgG;IAChG,gGAAgG;IAChG,iGAAiG;IACjG,8FAA8F;IAC9F,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,kGAAkG;IAClG,2FAA2F;IAC3F,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC;SACR,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;KACrD,CAAC;SACD,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC7E,CAAC,CAAC;AAEH;;2CAE2C;AAC3C,MAAM,mBAAmB,GAAG;IAC1B,cAAc;IACd,kBAAkB;IAClB,oBAAoB;IACpB,WAAW;IACX,eAAe;IACf,YAAY;IACZ,gBAAgB;IAChB,WAAW;IACX,OAAO;CACC,CAAC;AAEX;;2CAE2C;AAC3C,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAIrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,UAAmB;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,kCAAkC,MAAM,IAAI;YAC1C,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACvE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAEzB,qGAAqG;IACrG,oGAAoG;IACpG,IAAI,MAAkB,CAAC;IACvB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,iCAAiC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;gBACnF,mFAAmF;gBACnF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/D,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,mFAAmF;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,+DAA+D;gBACxF,4EAA4E;gBAC5E,4BAA4B,CAC/B,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,gEAAgE;gBACzF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC5F,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACxC,kGAAkG;QAClG,4FAA4F;QAC5F,2FAA2F;QAC3F,+EAA+E;QAC/E,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,0FAA0F;gBAC1F,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,mEAAmE;gBAC5F,6EAA6E;gBAC7E,2CAA2C,CAC9C,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8BAA8B,IAAI,CAAC,MAAM,CAAC,IAAI,mBAAmB;YAC1F,uFAAuF;YACvF,4FAA4F;YAC5F,oCAAoC,CACvC,CAAC;IACJ,CAAC;IAED,gGAAgG;IAChG,0FAA0F;IAC1F,yCAAyC;IACzC,IAAI,UAAgC,CAAC;IACrC,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,4FAA4F;YAC5F,yFAAyF;YACzF,sEAAsE;YACtE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,eAAe,IAAI,iCAAiC;oBAC7E,sFAAsF;oBACtF,uEAAuE,CAC1E,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,yBAAyB,IAAI,IAAI,CAAC,CAAC;YAChF,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IAC/B,CAAC;IACD,MAAM,kBAAkB,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAExE,oGAAoG;IACpG,kGAAkG;IAClG,kGAAkG;IAClG,uEAAuE;IACvE,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,kEAAkE;YAC3F,uFAAuF;YACvF,uEAAuE,CAC1E,CAAC;IACJ,CAAC;IAED,kGAAkG;IAClG,oGAAoG;IACpG,+FAA+F;IAC/F,8FAA8F;IAC9F,qFAAqF;IACrF,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;YACvF,uFAAuF;YACvF,yEAAyE,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,IAAI,2BAA2B,CAAC;IAE3F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,wBAAwB,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAExB,4FAA4F;QAC5F,iGAAiG;QACjG,iGAAiG;QACjG,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,6BAA6B;gBAC5F,qFAAqF,CACxF,CAAC;QACJ,CAAC;QAED,IAAI,KAAiB,CAAC;QACtB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,4FAA4F;YAC5F,6FAA6F;YAC7F,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;YACxD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,IAAI,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAE,OAAmC,CAAC,GAAG,CAAC,KAAK,SAAS,CACjE,CAAC;YACF,IAAI,kBAAkB,IAAI,gBAAgB,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,sBAAsB,OAAO,CAAC,EAAE,YAAY,KAAK,aAAa;oBACvF,qFAAqF;oBACrF,mDAAmD,CACtD,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;YAC/B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,iBAAiB;oBAChF,wEAAwE,CAC3E,CAAC;YACJ,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE;gBAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,UAAU,SAAS,GAAG;wBACrF,0CAA0C,CAC7C,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS;wBACT,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,8FAA8F;YAC9F,kDAAkD;YAClD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,mBAAmB;oBAClF,oEAAoE,CACvE,CAAC;YACJ,CAAC;YACD,KAAK,GAAG;gBACN;oBACE,IAAI,EAAE,OAAO,CAAC,MAAM;oBACpB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS,EAAE,SAAS;wBACpB,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK;YACL,aAAa,EAAE,OAAO,CAAC,cAAc,IAAI,qBAAqB;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,mGAAmG;IACnG,yFAAyF;IACzF,8FAA8F;IAC9F,6FAA6F;IAC7F,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC5C,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,QAAQ,CAAC,EAAE,0CAA0C;4BACzF,sFAAsF;4BACtF,sFAAsF;4BACtF,kFAAkF;4BAClF,6DAA6D,CAChE,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,2CAA2C;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IAC7D,mGAAmG;IACnG,6FAA6F;IAC7F,oGAAoG;IACpG,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,oBAAoB,YAAY,iCAAiC;YAC1F,+BAA+B,CAClC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM;QACN,YAAY;QACZ,UAAU;QACV,KAAK;KACN,CAAC;AACJ,CAAC;AAsBD;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAAC,GAAsB,EAAE,GAAgB;IACrE,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC;IAE5E,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;IAC3C,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,IAAI,CAChD,CAAC,GAAG,EAAE,EAAE,CAAE,GAA+B,CAAC,GAAG,CAAC,KAAK,SAAS,CAC7D,CAAC;IACF,IAAI,SAAS,IAAI,iBAAiB,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,kBAAkB,SAAS,qBAAqB;YACvF,sFAAsF;YACtF,8CAA8C,CACjD,CAAC;IACJ,CAAC;IAED,IAAI,YAA+B,CAAC;IACpC,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,SAAS,GAAG,GAAG,CAAC,MAAO,CAAC;QAC9B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;gBACtF,8BAA8B,CACjC,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CACjD,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE;YACxC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU;YACV,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gGAAgG;QAChG,YAAY,GAAG;YACb,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE;gBAC/B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,SAAS;gBACrB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;aAC3C,CAAC;SACH,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,mGAAmG;IACnG,0FAA0F;IAC1F,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACrF,CAAC;YACF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,8CAA8C;oBACrF,IAAI,QAAQ,2EAA2E;oBACvF,qBAAqB,QAAQ,sDAAsD;oBACnF,cAAc,CACjB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAeD;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,GAAkB,EAClB,eAAqC,EACrC,GAAuB;IAEvB,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GACT,GAAG,CAAC,UAAU,KAAK,SAAS;QAC1B,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE;QAC5D,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,iBAAiB,GAAG,CAAC,UAAU,EAAE,CAAC;IAEhG,+FAA+F;IAC/F,6FAA6F;IAC7F,2BAA2B;IAC3B,IAAI,UAAgC,CAAC;IACrC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,iDAAiD;gBACxF,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,IAAI,oBAAoB;oBACxF,iEAAiE,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;QACD,UAAU,GAAG,eAAe,CAAC;IAC/B,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;IAC3C,MAAM,cAAc,GAAG,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAClD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;IAE5C,iGAAiG;IACjG,4FAA4F;IAC5F,MAAM,cAAc,GAAG,CAAC,QAA8B,EAAE,KAAa,EAAY,EAAE,CACjF,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,mBAAmB,KAAK,WAAW;gBAC1E,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI;gBAC9B,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IACL,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAE1E,iGAAiG;IACjG,yFAAyF;IACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QACjD,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,KAAK,CAAC,IAAI,aAAa;gBACvF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,OAAO,WAAW;YAChB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE;YAChD,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GACb,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC3B,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnB,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,YAAY,CAAC,MAAM,GAAG,CAAC;QACvB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC;IAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;YACtF,2FAA2F;YAC3F,+EAA+E,CAClF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU;QACV,WAAW;QACX,cAAc;QACd,gBAAgB;QAChB,QAAQ;QACR,WAAW;QACX,SAAS;QACT,YAAY;QACZ,QAAQ;QACR,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;KAChD,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evalSuite.js","sourceRoot":"","sources":["../src/evalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4CAA4C,CAAC;IACrE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH;;oEAEoE;AACpE,MAAM,gCAAgC,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,gEAAgE,CAAC;IACzF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wDAAwD,CAAC;IACjF,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH;sGACsG;AACtG,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC5C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9C,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IACrD,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,qBAAqB,EAAE,CAAC,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC,QAAQ,EAAE;IAC3E,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,MAAM,CAAC;IACtD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH;;;mEAGmE;AACnE,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAC5C,KAAK,CACJ,WAAW,EACX,wFAAwF;QACtF,0FAA0F,CAC7F;IACH,uFAAuF;IACvF,8EAA8E;IAC9E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,iGAAiG;IACjG,mEAAmE;IACnE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;IAChD,mGAAmG;IACnG,+FAA+F;IAC/F,sDAAsD;IACtD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IACxC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC9B,0FAA0F;QAC1F,2FAA2F;QAC3F,mEAAmE;QACnE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC1B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAChC,CAAC;IACF,kGAAkG;IAClG,gGAAgG;IAChG,gGAAgG;IAChG,iGAAiG;IACjG,8FAA8F;IAC9F,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,kGAAkG;IAClG,2FAA2F;IAC3F,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC;SACR,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;KACrD,CAAC;SACD,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC7E,CAAC,CAAC;AAEH;;2CAE2C;AAC3C,MAAM,mBAAmB,GAAG;IAC1B,cAAc;IACd,kBAAkB;IAClB,oBAAoB;IACpB,WAAW;IACX,eAAe;IACf,YAAY;IACZ,gBAAgB;IAChB,WAAW;IACX,YAAY;IACZ,uBAAuB;IACvB,OAAO;CACC,CAAC;AAEX;;2CAE2C;AAC3C,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAIrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,UAAmB;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,kCAAkC,MAAM,IAAI;YAC1C,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACvE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAEzB,qGAAqG;IACrG,oGAAoG;IACpG,IAAI,MAAkB,CAAC;IACvB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,iCAAiC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;gBACnF,mFAAmF;gBACnF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/D,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,mFAAmF;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,+DAA+D;gBACxF,4EAA4E;gBAC5E,4BAA4B,CAC/B,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,gEAAgE;gBACzF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC5F,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACxC,kGAAkG;QAClG,4FAA4F;QAC5F,2FAA2F;QAC3F,+EAA+E;QAC/E,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,0FAA0F;gBAC1F,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,mEAAmE;gBAC5F,6EAA6E;gBAC7E,2CAA2C,CAC9C,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8BAA8B,IAAI,CAAC,MAAM,CAAC,IAAI,mBAAmB;YAC1F,uFAAuF;YACvF,4FAA4F;YAC5F,oCAAoC,CACvC,CAAC;IACJ,CAAC;IAED,gGAAgG;IAChG,0FAA0F;IAC1F,yCAAyC;IACzC,IAAI,UAAgC,CAAC;IACrC,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,4FAA4F;YAC5F,yFAAyF;YACzF,sEAAsE;YACtE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,eAAe,IAAI,iCAAiC;oBAC7E,sFAAsF;oBACtF,uEAAuE,CAC1E,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,yBAAyB,IAAI,IAAI,CAAC,CAAC;YAChF,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IAC/B,CAAC;IACD,MAAM,kBAAkB,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAExE,oGAAoG;IACpG,kGAAkG;IAClG,kGAAkG;IAClG,uEAAuE;IACvE,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,kEAAkE;YAC3F,uFAAuF;YACvF,uEAAuE,CAC1E,CAAC;IACJ,CAAC;IAED,kGAAkG;IAClG,oGAAoG;IACpG,+FAA+F;IAC/F,8FAA8F;IAC9F,qFAAqF;IACrF,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;YACvF,uFAAuF;YACvF,yEAAyE,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,IAAI,2BAA2B,CAAC;IAE3F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,wBAAwB,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAExB,4FAA4F;QAC5F,iGAAiG;QACjG,iGAAiG;QACjG,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,6BAA6B;gBAC5F,qFAAqF,CACxF,CAAC;QACJ,CAAC;QAED,IAAI,KAAiB,CAAC;QACtB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,4FAA4F;YAC5F,6FAA6F;YAC7F,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;YACxD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,IAAI,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAE,OAAmC,CAAC,GAAG,CAAC,KAAK,SAAS,CACjE,CAAC;YACF,IAAI,kBAAkB,IAAI,gBAAgB,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,sBAAsB,OAAO,CAAC,EAAE,YAAY,KAAK,aAAa;oBACvF,qFAAqF;oBACrF,mDAAmD,CACtD,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;YAC/B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,iBAAiB;oBAChF,wEAAwE,CAC3E,CAAC;YACJ,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE;gBAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,UAAU,SAAS,GAAG;wBACrF,0CAA0C,CAC7C,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS;wBACT,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,8FAA8F;YAC9F,kDAAkD;YAClD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,mBAAmB;oBAClF,oEAAoE,CACvE,CAAC;YACJ,CAAC;YACD,KAAK,GAAG;gBACN;oBACE,IAAI,EAAE,OAAO,CAAC,MAAM;oBACpB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS,EAAE,SAAS;wBACpB,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK;YACL,aAAa,EAAE,OAAO,CAAC,cAAc,IAAI,qBAAqB;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,mGAAmG;IACnG,yFAAyF;IACzF,8FAA8F;IAC9F,6FAA6F;IAC7F,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC5C,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,QAAQ,CAAC,EAAE,0CAA0C;4BACzF,sFAAsF;4BACtF,sFAAsF;4BACtF,kFAAkF;4BAClF,6DAA6D,CAChE,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,iGAAiG;IACjG,gGAAgG;IAChG,2FAA2F;IAC3F,+FAA+F;IAC/F,8FAA8F;IAC9F,gGAAgG;IAChG,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3D,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC5C,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAClF,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,QAAQ,CAAC,EAAE,wBAAwB;4BACvE,kFAAkF;4BAClF,qDACE,MAAM,CAAC,IAAI,KAAK,OAAO;gCACrB,CAAC,CAAC,qEAAqE;gCACvE,CAAC,CAAC,mDACN,+EAA+E,CAClF,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,2CAA2C;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IAC7D,mGAAmG;IACnG,6FAA6F;IAC7F,oGAAoG;IACpG,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,oBAAoB,YAAY,iCAAiC;YAC1F,+BAA+B,CAClC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM;QACN,YAAY;QACZ,UAAU;QACV,KAAK;KACN,CAAC;AACJ,CAAC;AAsBD;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAAC,GAAsB,EAAE,GAAgB;IACrE,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC;IAE5E,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;IAC3C,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,IAAI,CAChD,CAAC,GAAG,EAAE,EAAE,CAAE,GAA+B,CAAC,GAAG,CAAC,KAAK,SAAS,CAC7D,CAAC;IACF,IAAI,SAAS,IAAI,iBAAiB,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,kBAAkB,SAAS,qBAAqB;YACvF,sFAAsF;YACtF,8CAA8C,CACjD,CAAC;IACJ,CAAC;IAED,IAAI,YAA+B,CAAC;IACpC,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,SAAS,GAAG,GAAG,CAAC,MAAO,CAAC;QAC9B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;gBACtF,8BAA8B,CACjC,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CACjD,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE;YACxC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU;YACV,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gGAAgG;QAChG,YAAY,GAAG;YACb,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE;gBAC/B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,SAAS;gBACrB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;aAC3C,CAAC;SACH,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,mGAAmG;IACnG,0FAA0F;IAC1F,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACrF,CAAC;YACF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,8CAA8C;oBACrF,IAAI,QAAQ,2EAA2E;oBACvF,qBAAqB,QAAQ,sDAAsD;oBACnF,cAAc,CACjB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAeD;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,GAAkB,EAClB,eAAqC,EACrC,GAAuB;IAEvB,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GACT,GAAG,CAAC,UAAU,KAAK,SAAS;QAC1B,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE;QAC5D,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,iBAAiB,GAAG,CAAC,UAAU,EAAE,CAAC;IAEhG,+FAA+F;IAC/F,6FAA6F;IAC7F,2BAA2B;IAC3B,IAAI,UAAgC,CAAC;IACrC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,iDAAiD;gBACxF,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,IAAI,oBAAoB;oBACxF,iEAAiE,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;QACD,UAAU,GAAG,eAAe,CAAC;IAC/B,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;IAC3C,MAAM,cAAc,GAAG,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAClD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;IAE5C,iGAAiG;IACjG,4FAA4F;IAC5F,MAAM,cAAc,GAAG,CAAC,QAA8B,EAAE,KAAa,EAAY,EAAE,CACjF,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,mBAAmB,KAAK,WAAW;gBAC1E,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI;gBAC9B,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IACL,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAE1E,iGAAiG;IACjG,yFAAyF;IACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QACjD,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,KAAK,CAAC,IAAI,aAAa;gBACvF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,OAAO,WAAW;YAChB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE;YAChD,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,0FAA0F;IAC1F,qFAAqF;IACrF,2FAA2F;IAC3F,+FAA+F;IAC/F,2EAA2E;IAC3E,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;IACvC,MAAM,kBAAkB,GAAG,CAAC,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACzE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QACjD,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,mCAAmC;gBAC1E,IAAI,KAAK,CAAC,IAAI,+EAA+E,CAChG,CAAC;QACJ,CAAC;QACD,IAAI,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzF,IAAI,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QACnF,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GACb,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC3B,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnB,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,YAAY,CAAC,MAAM,GAAG,CAAC;QACvB,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnB,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC;IAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;YACtF,2FAA2F;YAC3F,2EAA2E;YAC3E,2CAA2C,CAC9C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU;QACV,WAAW;QACX,cAAc;QACd,gBAAgB;QAChB,QAAQ;QACR,WAAW;QACX,SAAS;QACT,YAAY;QACZ,QAAQ;QACR,SAAS;QACT,kBAAkB;QAClB,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;KAChD,CAAC;AACJ,CAAC"}

package/dist/evalTypes.d.ts

@@ -5,6 +5,7 @@
  * (BATCH-1's cell/outcome shapes), which documents itself as scoped to "cells and outcomes" only
  * — eval's shapes layer on top of (not into) that file.
  */
+import type { ToolResultRecord } from '#src/types.js';
 /** The 0-10 judge scale's default pass threshold, matching `review`'s own (unexported)
  * `DEFAULT_PASS_THRESHOLD` in `packages/review/src/middleware/reviewRateMiddleware.ts` — same
  * scale, same default, so a user who knows `review`'s threshold semantics already knows eval's.
@@ -78,6 +79,25 @@ export interface JsonPathCheck {
     equals?: unknown;
     contains?: string;
 }
+/**
+ * One `tool_result_json_path` assertion (BATCH-21): select tool results by name `tool` (exact or
+ * glob, the same matcher `must_call` uses), parse each matching result's payload as JSON, and
+ * resolve `path` against it (the same minimal dot/`[index]` path `json_path` uses). At most one of
+ * `equals`/`contains` may be set (enforced in {@link ../evalSuite.js}'s parse):
+ * - neither — pure existence check: the path must resolve in a matching result's payload;
+ * - `equals` — the resolved value must deep-equal this (any JSON value, incl. `null`);
+ * - `contains` — the resolved value must be a string containing this substring.
+ * The assertion passes when AT LEAST ONE matching tool result satisfies it; a non-JSON payload is a
+ * deterministic FAIL for that result, never a throw. Requires the `gth-agent` target (rejected at
+ * parse time otherwise — only the in-process agent surfaces tool results).
+ */
+export interface ToolResultJsonPathCheck {
+    /** Tool-name pattern (exact or glob) selecting which tool's result(s) to check. */
+    tool: string;
+    path: string;
+    equals?: unknown;
+    contains?: string;
+}
 /**
  * One expectation block (BATCH-12) — the BATCH-10 assertion bundle PLUS an optional `identities`
  * scope. This is the atom the whole eval layer normalizes to: a flat case is sugar for ONE
@@ -107,6 +127,13 @@ export interface EvalExpectation {
     mustNotMatch: RegExp[];
     /** BATCH-10 minimal JSON-path assertions over the answer parsed as JSON — see {@link JsonPathCheck}. */
     jsonPath: JsonPathCheck[];
+    /** BATCH-21 tool-RESULT assertion: for each tool-name pattern (exact/glob, `must_call`'s
+     * matcher), at least one called tool matching it returned an ERROR (`isError: true`). Grades the
+     * captured tool results, so it requires the `gth-agent` target (parse-rejected otherwise). */
+    mustError: string[];
+    /** BATCH-21 minimal JSON-path assertions over a matching tool result's payload — see
+     * {@link ToolResultJsonPathCheck}. `gth-agent` target only (parse-rejected otherwise). */
+    toolResultJsonPath: ToolResultJsonPathCheck[];
     /** The judge rubric, when present and non-blank. `undefined` = no judge for this block. */
     judgeRubric?: string;
 }
@@ -133,6 +160,9 @@ export interface TurnRunOutcome {
     tokensInput?: number;
     tokensOutput?: number;
     tools?: string[];
+    /** BATCH-21 — this turn's per-tool-call result records (parallel to {@link tools}; a per-turn
+     * delta like everything else here). Only the in-process `gth-agent` runner populates it. */
+    toolResults?: ToolResultRecord[];
     error?: string;
 }
 /**
@@ -218,6 +248,8 @@ export interface EvalTurnResult {
     tokensInput?: number;
     tokensOutput?: number;
     tools?: string[];
+    /** BATCH-21 — this turn's per-tool-call result records (mirrors {@link TurnRunOutcome.toolResults}). */
+    toolResults?: ToolResultRecord[];
     /** Whether this turn's SUT invocation ran (mirrors {@link TurnRunOutcome.ok}); `false` = the turn
      * produced no answer (transport/agent error, or the conversation aborted before reaching it). */
     ok: boolean;
@@ -245,6 +277,10 @@ export interface EvalCaseResult {
     tokensInput?: number;
     tokensOutput?: number;
     tools?: string[];
+    /** BATCH-21 — the cell's per-tool-call result records (parallel to {@link tools}; omitted when
+     * the runner produced none, so a pre-BATCH-21 `<id>.json` stays byte-for-byte identical). For a
+     * multi-turn cell this stays unset like `tools` — read {@link turns} instead. */
+    toolResults?: ToolResultRecord[];
     durationMs: number;
     checks?: DeterministicCheckResult;
     judge?: JudgeOutcome;

package/dist/evalTypes.js

@@ -1,10 +1,3 @@
-/**
- * @packageDocumentation
- * BATCH-2 — the shapes for `gth eval`: a parsed suite/case, deterministic-check results, the
- * judge's verdict, and per-case/suite outcomes. Deliberately separate from {@link ../types.js}
- * (BATCH-1's cell/outcome shapes), which documents itself as scoped to "cells and outcomes" only
- * — eval's shapes layer on top of (not into) that file.
- */
 /** The 0-10 judge scale's default pass threshold, matching `review`'s own (unexported)
  * `DEFAULT_PASS_THRESHOLD` in `packages/review/src/middleware/reviewRateMiddleware.ts` — same
  * scale, same default, so a user who knows `review`'s threshold semantics already knows eval's.

package/dist/evalTypes.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalTypes.js","sourceRoot":"","sources":["../src/evalTypes.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;;;mGAImG;AACnG,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"evalTypes.js","sourceRoot":"","sources":["../src/evalTypes.ts"],"names":[],"mappings":"AASA;;;;mGAImG;AACnG,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}

package/dist/index.d.ts

@@ -3,7 +3,7 @@ export { bindCellContent } from '#src/interpolate.js';
 export { parseOverFile } from '#src/parseOver.js';
 export { runBatchMatrix, buildBatchSummary } from '#src/BatchRunner.js';
 export { writeBatchOutput } from '#src/output.js';
-export type { BatchRunnerOptions, BatchSummary, CellResult, CellRunOutcome, MatrixCell, MatrixRow, RunCellFn, } from '#src/types.js';
+export type { BatchRunnerOptions, BatchSummary, CellResult, CellRunOutcome, MatrixCell, MatrixRow, RunCellFn, ToolResultRecord, } from '#src/types.js';
 export { DEFAULT_CONCURRENCY } from '#src/types.js';
 export { parseEvalSuite } from '#src/evalSuite.js';
 export { runDeterministicChecks } from '#src/deterministicChecks.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAWlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}

package/dist/toolResultChecks.d.ts

@@ -0,0 +1,30 @@
+import type { EvalExpectation } from '#src/evalTypes.js';
+import type { ToolResultRecord } from '#src/types.js';
+/**
+ * BATCH-21 tool-RESULT assertions — grade a cell against what its called tools *returned*
+ * (`toolResults`: per-call `isError` + payload), not just which names were called (BATCH-10's
+ * `#src/toolChecks.js`) or what the answer said. This is the structural check the authz/data-
+ * isolation suites need: "the restricted identity CALLED the tool AND the call came back denied"
+ * becomes deterministic instead of judge-graded.
+ *
+ * Kept beside (not inside) `runToolCallChecks` on purpose — same file-per-input-kind split the
+ * answer checks use: these read the tool RESULTS, that one reads the tool NAMES. Name patterns
+ * reuse the same `toolNameMatchesPattern` glob/exact matcher (`@gaunt-sloth/core`), so
+ * `mcp__unimarket__*` selects results exactly as it selects calls in `must_call`; payload paths
+ * reuse `resolveJsonPath`, the same minimal dot/`[index]` evaluator `json_path` uses on the answer.
+ *
+ * - `mustError` — for **each** pattern, at least one tool result whose name matches it has
+ *   `isError: true`, else `tool "<pattern>" did not return an error` (the same single message
+ *   whether the tool succeeded or was never called — either way, no matching error came back).
+ * - `toolResultJsonPath` — each entry passes iff **at least one** result from a tool matching its
+ *   `tool` pattern satisfies it: the payload parses as JSON, the `path` resolves, and (when set)
+ *   `equals` deep-equals / `contains` substring-matches the resolved value. A non-JSON (or absent)
+ *   payload is a deterministic per-result FAIL, never a throw — note capture caps payloads
+ *   (`TOOL_RESULT_CONTENT_CAP`), so a truncated over-cap payload also fails as non-JSON.
+ *
+ * The runner only ever calls this with expectation blocks a `gth-agent` suite produced — the suite
+ * parser rejects tool-result assertions against `ag-ui`/`adk-agent` targets, whose wire carries no
+ * result payloads (see `#src/evalSuite.js`), so an empty `toolResults` here means "no tool
+ * returned anything", not "the target can't report results".
+ */
+export declare function runToolResultChecks(toolResults: ToolResultRecord[], expectation: Pick<EvalExpectation, 'mustError' | 'toolResultJsonPath'>): string[];

package/dist/toolResultChecks.js

@@ -0,0 +1,102 @@
+import { isDeepStrictEqual } from 'node:util';
+import { toolNameMatchesPattern } from '@gaunt-sloth/core/utils/toolMatching.js';
+import { resolveJsonPath } from '#src/deterministicChecks.js';
+/**
+ * BATCH-21 tool-RESULT assertions — grade a cell against what its called tools *returned*
+ * (`toolResults`: per-call `isError` + payload), not just which names were called (BATCH-10's
+ * `#src/toolChecks.js`) or what the answer said. This is the structural check the authz/data-
+ * isolation suites need: "the restricted identity CALLED the tool AND the call came back denied"
+ * becomes deterministic instead of judge-graded.
+ *
+ * Kept beside (not inside) `runToolCallChecks` on purpose — same file-per-input-kind split the
+ * answer checks use: these read the tool RESULTS, that one reads the tool NAMES. Name patterns
+ * reuse the same `toolNameMatchesPattern` glob/exact matcher (`@gaunt-sloth/core`), so
+ * `mcp__unimarket__*` selects results exactly as it selects calls in `must_call`; payload paths
+ * reuse `resolveJsonPath`, the same minimal dot/`[index]` evaluator `json_path` uses on the answer.
+ *
+ * - `mustError` — for **each** pattern, at least one tool result whose name matches it has
+ *   `isError: true`, else `tool "<pattern>" did not return an error` (the same single message
+ *   whether the tool succeeded or was never called — either way, no matching error came back).
+ * - `toolResultJsonPath` — each entry passes iff **at least one** result from a tool matching its
+ *   `tool` pattern satisfies it: the payload parses as JSON, the `path` resolves, and (when set)
+ *   `equals` deep-equals / `contains` substring-matches the resolved value. A non-JSON (or absent)
+ *   payload is a deterministic per-result FAIL, never a throw — note capture caps payloads
+ *   (`TOOL_RESULT_CONTENT_CAP`), so a truncated over-cap payload also fails as non-JSON.
+ *
+ * The runner only ever calls this with expectation blocks a `gth-agent` suite produced — the suite
+ * parser rejects tool-result assertions against `ag-ui`/`adk-agent` targets, whose wire carries no
+ * result payloads (see `#src/evalSuite.js`), so an empty `toolResults` here means "no tool
+ * returned anything", not "the target can't report results".
+ */
+export function runToolResultChecks(toolResults, expectation) {
+    const failures = [];
+    for (const pattern of expectation.mustError) {
+        const matchedAnError = toolResults.some((result) => result.isError && toolNameMatchesPattern(result.name, pattern));
+        if (!matchedAnError) {
+            failures.push(`tool "${pattern}" did not return an error`);
+        }
+    }
+    for (const check of expectation.toolResultJsonPath) {
+        const failure = checkToolResultJsonPath(toolResults, check);
+        if (failure !== undefined) {
+            failures.push(failure);
+        }
+    }
+    return failures;
+}
+/** Grade one {@link ToolResultJsonPathCheck}: `undefined` when at least one matching tool result
+ * satisfies it, else ONE failure line (per check, not per result) naming the path + tool pattern
+ * and the distinct per-result reasons. */
+function checkToolResultJsonPath(toolResults, check) {
+    const label = `tool_result_json_path "${check.path}" (tool "${check.tool}")`;
+    const matching = toolResults.filter((result) => toolNameMatchesPattern(result.name, check.tool));
+    if (matching.length === 0) {
+        return `${label}: no result from a matching tool`;
+    }
+    // First-seen-order distinct reasons: with several matching results (a tool called N times), a
+    // repeated reason is reported once, and ANY passing result clears the whole check.
+    const reasons = new Set();
+    for (const result of matching) {
+        const reason = evaluateResultAgainstCheck(result, check);
+        if (reason === undefined)
+            return undefined;
+        reasons.add(reason);
+    }
+    return `${label}: ${[...reasons].join('; ')}`;
+}
+/** Evaluate ONE tool result against ONE check: `undefined` = satisfied, else the failure reason.
+ * Deterministic and throw-free: a non-JSON/absent payload is a reason, not an exception. */
+function evaluateResultAgainstCheck(result, check) {
+    let root;
+    try {
+        root = JSON.parse((result.content ?? '').trim());
+    }
+    catch {
+        return 'result payload is not JSON';
+    }
+    const { found, value } = resolveJsonPath(root, check.path);
+    if (!found) {
+        return 'path did not resolve';
+    }
+    if (check.contains !== undefined) {
+        if (typeof value !== 'string') {
+            return `is ${JSON.stringify(value)} (contains check requires a string)`;
+        }
+        if (!value.includes(check.contains)) {
+            return `does not contain "${check.contains}"`;
+        }
+        return undefined;
+    }
+    // `equals` may legitimately be `null`, so discriminate on KEY presence (the parser normalizes
+    // the key out entirely for a pure existence check, and keeps it — possibly null-valued — when
+    // the suite set it).
+    if ('equals' in check) {
+        if (!isDeepStrictEqual(value, check.equals)) {
+            return `is ${JSON.stringify(value)}, expected ${JSON.stringify(check.equals)}`;
+        }
+        return undefined;
+    }
+    // Neither equals nor contains: a pure existence check — the resolved path is enough.
+    return undefined;
+}
+//# sourceMappingURL=toolResultChecks.js.map

package/dist/toolResultChecks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolResultChecks.js","sourceRoot":"","sources":["../src/toolResultChecks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAI9D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,mBAAmB,CACjC,WAA+B,EAC/B,WAAsE;IAEtE,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAC3E,CAAC;QACF,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,SAAS,OAAO,2BAA2B,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,uBAAuB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;0CAE0C;AAC1C,SAAS,uBAAuB,CAC9B,WAA+B,EAC/B,KAA8B;IAE9B,MAAM,KAAK,GAAG,0BAA0B,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC,IAAI,IAAI,CAAC;IAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,KAAK,kCAAkC,CAAC;IACpD,CAAC;IAED,8FAA8F;IAC9F,mFAAmF;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,0BAA0B,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,GAAG,KAAK,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAChD,CAAC;AAED;4FAC4F;AAC5F,SAAS,0BAA0B,CACjC,MAAwB,EACxB,KAA8B;IAE9B,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,4BAA4B,CAAC;IACtC,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,qCAAqC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO,qBAAqB,KAAK,CAAC,QAAQ,GAAG,CAAC;QAChD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8FAA8F;IAC9F,8FAA8F;IAC9F,qBAAqB;IACrB,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACjF,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,qFAAqF;IACrF,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/types.d.ts

@@ -33,6 +33,20 @@ export interface MatrixCell {
     /** The fully-bound prompt content for this cell (base script content with the row interpolated). */
     content: string;
 }
+/**
+ * BATCH-21 — one executed tool call's result record, as captured by core's GS2-16 run-stats
+ * accumulator (a structural mirror of `@gaunt-sloth/core`'s `GthToolResult`, kept local so this
+ * package's cell/outcome shapes stay independent of LLM/runner types — same reason `tools` is a
+ * plain `string[]` here). One record per executed tool call, in arrival order, NOT deduplicated.
+ */
+export interface ToolResultRecord {
+    /** The tool that produced the result. */
+    name: string;
+    /** `true` iff the tool result carried the error status (LangChain `ToolMessage.status === 'error'`). */
+    isError: boolean;
+    /** The result payload as text (non-string payloads JSON-stringified), size-capped at capture. */
+    content?: string;
+}
 /** What one attempt at running a cell through the shared single-shot runtime produced. */
 export interface CellRunOutcome {
     /** `true` when the cell's run completed without error; mirrors `runSingleShot`'s contract. */
@@ -50,6 +64,10 @@ export interface CellRunOutcome {
     tokensOutput?: number;
     /** Names of tools invoked during the run, when available (see {@link answer}'s caveat). */
     tools?: string[];
+    /** BATCH-21 — per-tool-call result records for the run, when available; parallel to
+     * {@link tools} but un-deduped and carrying each result's error status + payload. Only the
+     * in-process `gth-agent` runner can populate this (external adk-agent/ag-ui targets never do). */
+    toolResults?: ToolResultRecord[];
     /** A human-readable failure reason, set when `ok` is `false`. */
     error?: string;
 }

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA+FH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkHH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gaunt-sloth/batch",
-  "version": "2.0.0-alpha.21",
+  "version": "2.0.0-alpha.23",
   "description": "Batch matrix runtime for Gaunt Sloth (BATCH-1): run a prompt-executable over a matrix of models and/or content-bound inputs",
   "license": "MIT",
   "author": "Andrew Kondratev",
@@ -37,8 +37,8 @@
     "@langchain/core": "^1.2.3",
     "yaml": "^2.9.0",
     "zod": "^4.4.3",
-    "@gaunt-sloth/agent": "2.0.0-alpha.21",
-    "@gaunt-sloth/core": "2.0.0-alpha.21"
+    "@gaunt-sloth/agent": "2.0.0-alpha.23",
+    "@gaunt-sloth/core": "2.0.0-alpha.23"
   },
   "files": [
     "./dist/*"