@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.26

package/dist/evalRunner.js

@@ -2,6 +2,22 @@ import { runBatchMatrix } from '#src/BatchRunner.js';
 import { runDeterministicChecks } from '#src/deterministicChecks.js';
 import { runToolCallChecks } from '#src/toolChecks.js';
 import { runToolResultChecks } from '#src/toolResultChecks.js';
+import { mechanismNeedsPermissiveRating } from '#src/evalTypes.js';
+import { UNRECOGNIZED_LABEL } from '#src/classificationTypes.js';
+import { extractClassificationValue, readRaw } from '#src/classification.js';
+import { buildClassificationReport } from '#src/classificationReport.js';
+/**
+ * BATCH-25 — a case's tags, read defensively.
+ *
+ * `EvalCase.tags` is REQUIRED and `parseEvalSuite` always supplies it (`[]` when the case declares
+ * none), following this layer's stated convention that arrays default to `[]` so callers never need
+ * an existence check. But `EvalSuite` is a public export that a caller can also construct by hand,
+ * and a missing array must degrade to "no tags" rather than crash a whole run — the same `?? []`
+ * discipline `cellResult.tools`/`toolResults` already get here.
+ */
+function caseTags(evalCase) {
+    return evalCase.tags ?? [];
+}
 /** A block applies to an identity when it names no identities (unscoped → all) or explicitly names
  * this one. For the single-run pseudo-identity (`undefined`) only unscoped blocks apply — and the
  * parser guarantees a no-identities suite has only unscoped blocks. */
@@ -48,6 +64,16 @@ function buildUnits(suite) {
  * single-turn unit keeps the proven `runCell` + {@link gradeUnit} path byte-for-byte.
  */
 export async function runEvalSuite(suite, options) {
+    // BATCH-25 Half B — a classification target that never arrived must be LOUD. `options.classify`
+    // is optional (a classifier suite can read its label out of an agent answer), so a `rater` suite
+    // whose command forgot to build the classifier would otherwise fall through to `runCell` and send
+    // every shell command to the chat model as a prompt — a full run, billed, graded against prose,
+    // and reported as if it had measured the rater. Fail before anything runs instead.
+    if (suite.target.type === 'rater' && !options.classify) {
+        throw new Error('runEvalSuite: a "rater" target needs an injected `classify` function — build it with ' +
+            '`buildRaterClassifier(suite.target, config)` (#src/raterTarget.js) and pass it as ' +
+            '`options.classify`. Without it every case would be run through the ordinary agent.');
+    }
     const units = buildUnits(suite);
     // I1 — dispatch AND grading are keyed by the guaranteed-unique `inputIndex` (a unit's position in
     // this array, threaded onto its `MatrixCell`/`CellResult`), NEVER by the composite `cellId`
@@ -103,8 +129,50 @@ export async function runEvalSuite(suite, options) {
     // cells — `CellRunOutcome` carries one answer, a conversation carries N). Only multi-turn cells
     // populate this; single-turn cells grade straight from their `CellResult` as before.
     const conversationOutcomes = new Map();
+    // The classification target is only in play for a suite that actually declares a `classification:`
+    // block — an injected classifier with no label enum would have no axes to report against.
+    const spec = suite.classification;
+    const classifier = spec ? options.classify : undefined;
+    // BATCH-25 — per-round classification outcomes from an injected classification target, keyed by
+    // the same unique `inputIndex`. Populated only on the `options.classify` path.
+    const classifyOutcomes = new Map();
     const dispatchRunCell = async (cell) => {
         const unit = unitByInputIndex.get(cell.inputIndex);
+        // BATCH-25 — a CLASSIFICATION target decides the case itself; there is no agent turn to run.
+        // Its per-round outcomes are stashed for grading and ALSO adapted into the conversation stash,
+        // so a multi-round classifier case rides the existing per-turn grader unchanged.
+        if (classifier) {
+            const outcomes = await classifier({
+                caseId: unit.evalCase.id,
+                inputs: unit.evalCase.turns.map((turn) => turn.user),
+                tags: caseTags(unit.evalCase),
+                modelFree: unit.evalCase.modelFree,
+                // Passed on as DECLARED, per round. The runner is target-agnostic and forms no opinion
+                // about what a mechanism implies — `ClassifyRequest.forcedBy` says why that split matters.
+                //
+                // Where one round declares SEVERAL mechanisms (`ob-05` is the corpus case: an ambiguity
+                // preflight AND the floor), a mechanism needing a permissive rating is preferred over one
+                // that does not, rather than simply taking the first block. Taking the first made the
+                // verdict depend on BLOCK ORDER — `[ambiguity, floor]` passed and `[floor, ambiguity]`
+                // failed on identical assertions — because a round left undriven can never let its
+                // preflight speak. Preferring the preflight satisfies BOTH claims: the floor's marker comes
+                // from `checkHardline`, which is unaffected by the rating either way.
+                forcedBy: unit.evalCase.turns.map((turn) => turn.expectations.find((block) => mechanismNeedsPermissiveRating(block.forcedBy))
+                    ?.forcedBy ??
+                    turn.expectations.find((block) => block.forcedBy !== undefined)?.forcedBy),
+            });
+            classifyOutcomes.set(cell.inputIndex, outcomes);
+            conversationOutcomes.set(cell.inputIndex, outcomes.map((outcome) => ({
+                ok: outcome.ok,
+                answer: outcome.rationale,
+                error: outcome.error,
+            })));
+            return {
+                ok: outcomes.length > 0 && outcomes.every((outcome) => outcome.ok),
+                answer: outcomes[0]?.rationale,
+                error: outcomes[0]?.error,
+            };
+        }
         // Single-turn: the proven `runSingleShot`-backed path, byte-for-byte (unchanged dispatch).
         if (unit.evalCase.turns.length <= 1) {
             return runCellFor(unit.identity)(cell);
@@ -127,15 +195,165 @@ export async function runEvalSuite(suite, options) {
            unit's array position, preserved through runBatchMatrix), so every result maps to one unit */
         if (!unit)
             continue;
+        const outcomes = conversationOutcomes.get(cellResult.inputIndex);
+        // BATCH-25 — the per-round answers this unit produced, in turn order. A single-turn cell has one
+        // (the cell's own answer); a multi-turn / classifier cell has one per round.
+        const answersPerTurn = unit.evalCase.turns.length <= 1
+            ? [cellResult.answer]
+            : unit.evalCase.turns.map((_, index) => outcomes?.[index]?.answer);
+        const classifications = buildActualClassifications(unit, spec, answersPerTurn, classifyOutcomes.get(cellResult.inputIndex));
         if (unit.evalCase.turns.length <= 1) {
-            results.push(await gradeUnit(unit, cellResult, options.judge));
+            results.push(await gradeUnit(unit, cellResult, options.judge, classifications?.[0]));
         }
         else {
-            results.push(await gradeConversationUnit(unit, conversationOutcomes.get(cellResult.inputIndex), cellResult, options.judge));
+            results.push(await gradeConversationUnit(unit, outcomes, cellResult, options.judge, classifications));
         }
     }
     const passed = results.filter((result) => result.verdict === 'PASS').length;
-    return { total: results.length, passed, failed: results.length - passed, cases: results };
+    const summary = {
+        total: results.length,
+        passed,
+        failed: results.length - passed,
+        cases: results,
+    };
+    // BATCH-25 — the classifier report (matrices + declared metrics) is attached ONLY for a suite that
+    // declares `classification:`, so a #405-era `results.json` is byte-for-byte unchanged.
+    if (spec) {
+        summary.classification = buildClassificationReport(spec, 
+        // Same defensive read as `caseTags`: required on the parsed shape, tolerated absent on a
+        // hand-constructed one.
+        suite.metrics ?? [], toClassifiedCells(results));
+    }
+    return summary;
+}
+/**
+ * BATCH-25 — reduce the graded results to the shape the matrices and metrics read.
+ *
+ * `scored` is the load-bearing field: a cell whose SUT never ran has NO place in any matrix cell or
+ * any metric denominator, and is instead counted as `excluded` and said out loud. Silently treating
+ * it as a wrong answer would inflate the error rate; silently dropping it would inflate coverage.
+ * Both are how a classifier report comes to claim more than it measured.
+ */
+export function toClassifiedCells(results) {
+    return results.map((result) => ({
+        id: result.identity === undefined ? result.id : `${result.id}__${result.identity}`,
+        tags: result.tags ?? [],
+        expectedLabel: result.classification?.expectedLabel,
+        expectedAction: result.classification?.expectedAction,
+        actualLabel: result.classification?.actualLabel,
+        actualAction: result.classification?.actualAction,
+        scored: result.sutOk && result.classification !== undefined,
+    }));
+}
+/**
+ * BATCH-25 — build one {@link EvalCaseClassification} per round: what the corpus expected, and what
+ * the SUT actually produced.
+ *
+ * The EXPECTED side comes from the round's APPLICABLE expectation blocks (already filtered to this
+ * identity), so an identity-scoped or round-scoped expectation is read correctly. When two
+ * applicable blocks declare different expected labels the first wins for reporting purposes — but
+ * both are still graded independently, so a contradictory pair fails the case rather than being
+ * quietly reconciled here.
+ *
+ * The ACTUAL side comes from the injected classification target when there is one, and otherwise
+ * from the suite's declared extractors reading the round's answer. Either way a value the suite does
+ * not declare becomes {@link UNRECOGNIZED_LABEL} — a visible bucket, never a dropped row.
+ */
+function buildActualClassifications(unit, spec, answersPerTurn, classifyOutcomes) {
+    if (!spec)
+        return undefined;
+    return unit.evalCase.turns.map((_turn, index) => {
+        const applicable = unit.applicablePerTurn[index] ?? [];
+        const expectedLabel = applicable.find((block) => block.expectLabel !== undefined)?.expectLabel;
+        const expectedAction = applicable.find((block) => block.expectAction !== undefined)?.expectAction;
+        const outcome = classifyOutcomes?.[index];
+        const answer = answersPerTurn[index];
+        let actualLabel;
+        let actualAction;
+        let raw;
+        if (outcome) {
+            // A classification target reports its verdict directly — but it is still held to the suite's
+            // declared enum, so a target returning an undeclared value shows up as `(unrecognized)`
+            // rather than silently widening the axis.
+            actualLabel = outcome.ok ? normalizeAgainstEnum(outcome.label, spec.labels) : undefined;
+            actualAction =
+                outcome.ok && spec.actions.length > 0
+                    ? normalizeAgainstEnum(outcome.action, spec.actions)
+                    : undefined;
+            raw = outcome.rationale;
+        }
+        else if (answer !== undefined) {
+            actualLabel = extractClassificationValue(answer, spec.labelFrom, spec.labels);
+            if (spec.actionFrom) {
+                actualAction = extractClassificationValue(answer, spec.actionFrom, spec.actions);
+            }
+            // Keep the text the extractor actually read when it is NOT just the whole answer, so an
+            // `(unrecognized)` result is diagnosable from the per-cell JSON without re-running.
+            const readValue = readRaw(answer, spec.labelFrom);
+            if (readValue !== undefined && readValue !== answer)
+                raw = readValue;
+        }
+        return {
+            expectedLabel,
+            expectedAction,
+            actualLabel,
+            actualAction,
+            raw,
+            modelCalls: outcome?.modelCalls,
+        };
+    });
+}
+/** Hold a classification target's own verdict to the suite's declared enum. An absent value stays
+ * absent (the target reported nothing on this dimension); an undeclared one becomes
+ * {@link UNRECOGNIZED_LABEL}. */
+function normalizeAgainstEnum(value, declared) {
+    if (value === undefined)
+        return undefined;
+    return declared.includes(value) ? value : UNRECOGNIZED_LABEL;
+}
+/**
+ * BATCH-25 — grade one round's classification assertions and the `model_free` contract.
+ *
+ * Returns failure reasons; empty means the round's classification assertions held. The label and
+ * the action are graded SEPARATELY and both are reported, because they diverge by design: a
+ * deterministic preflight can produce the right action from the wrong label, and a report that
+ * collapsed them would hide which component drifted.
+ */
+function gradeClassificationBlock(block, classification) {
+    const reasons = [];
+    if (block.expectLabel !== undefined) {
+        const actual = classification?.actualLabel;
+        if (actual !== block.expectLabel) {
+            reasons.push(`expected label "${block.expectLabel}" but got "${actual ?? '(none)'}"`);
+        }
+    }
+    if (block.expectAction !== undefined) {
+        const actual = classification?.actualAction;
+        if (actual !== block.expectAction) {
+            reasons.push(`expected action "${block.expectAction}" but got "${actual ?? '(none)'}"`);
+        }
+    }
+    return reasons;
+}
+/** BATCH-25 — the `model_free` contract: a case declared free must cost ZERO model calls. A target
+ * that quietly rings the model on such a case would otherwise be invisible, and these cases are
+ * exactly the cheap, deterministic regression gate (the EXT-55 class — a hardline floor that had
+ * silently stopped firing, invisible to build, lint and unit). `undefined` model calls means the
+ * target did not report them, which is itself a failure of the contract. */
+function gradeModelFree(evalCase, classification) {
+    if (!evalCase.modelFree)
+        return [];
+    const calls = classification?.modelCalls;
+    if (calls === undefined) {
+        return [
+            'model_free: the target reported no model-call count, so "zero model calls" could not be ' +
+                'verified.',
+        ];
+    }
+    if (calls > 0) {
+        return [`model_free: expected 0 model calls but the target made ${calls}`];
+    }
+    return [];
 }
 /**
  * BATCH-11 (#405 his #6) — classify a completed suite's {@link EvalSuiteSummary} into a distinct
@@ -153,6 +371,12 @@ export async function runEvalSuite(suite, options) {
  * Classification is anchored on `sutOk`, not the verdict: a cell that ran (`sutOk === true`) but
  * whose judge errored (or whose answer failed a check) is a real result → exit `1`, never `2`. A
  * *mix* of `sutOk:false` and `sutOk:true` cells therefore yields `1`.
+ *
+ * BATCH-25 adds ONE further way to reach `1`: a breached `gate: fail` metric threshold. That is a
+ * product signal of exactly the same kind — and it is the one that a per-case pass/fail sweep cannot
+ * express, because a classifier corpus can be entirely within per-case tolerance while its
+ * false-approve rate is unacceptable. Per-case verdicts answer "did each case behave"; a gated
+ * metric answers "is the aggregate shippable", and only the second gates a release.
  */
 export function classifyEvalExit(summary) {
     // No gradeable results at all → harness/environment error, not a product signal.
@@ -163,6 +387,11 @@ export function classifyEvalExit(summary) {
     if (summary.failed > 0) {
         return 1;
     }
+    // BATCH-25 — every cell passed, but a declared metric breached its hard threshold. Still a
+    // product signal (exit 1), never a harness one: the run worked, the aggregate is out of bounds.
+    if ((summary.classification?.gateFailures.length ?? 0) > 0) {
+        return 1;
+    }
     // Every cell passed.
     return 0;
 }
@@ -175,7 +404,7 @@ export function classifyEvalExit(summary) {
  * `judge` field is exactly that block's outcome (back-compat). For a single applicable block this
  * reproduces the pre-BATCH-12 grading byte-for-byte.
  */
-async function gradeApplicableBlocks(answer, tools, toolResults, applicable, passThreshold, judge) {
+async function gradeApplicableBlocks(answer, tools, toolResults, applicable, passThreshold, judge, classification) {
     const deterministicFailures = [];
     const reasons = [];
     let judgeOutcome;
@@ -188,8 +417,13 @@ async function gradeApplicableBlocks(answer, tools, toolResults, applicable, pas
         // third input kind, graded by its own checker (#src/toolResultChecks.js) and merged into the
         // same `reasons` so they drive the same PASS/FAIL/exit contract.
         const toolResultFailures = runToolResultChecks(toolResults, block);
+        // BATCH-25 classification assertions read the cell's CLASSIFICATION (a fourth input kind,
+        // beside the answer, the tool trace and the tool results), so they are graded by their own
+        // checker and merged into the same `reasons` — the same PASS/FAIL/exit contract as every other
+        // assertion.
+        const classificationFailures = gradeClassificationBlock(block, classification);
         deterministicFailures.push(...checks.failures);
-        reasons.push(...checks.failures, ...toolFailures, ...toolResultFailures);
+        reasons.push(...checks.failures, ...toolFailures, ...toolResultFailures, ...classificationFailures);
         if (block.judgeRubric) {
             let outcome;
             if (!judge) {
@@ -218,7 +452,7 @@ async function gradeApplicableBlocks(answer, tools, toolResults, applicable, pas
  * block passes. For a flat case (exactly one unscoped applicable block) this produces byte-for-byte
  * the same `reasons`/`checks`/`judge`/`verdict` as the pre-BATCH-12 single-block grader.
  */
-async function gradeUnit(unit, cellResult, judge) {
+async function gradeUnit(unit, cellResult, judge, classification) {
     const { evalCase, identity, applicablePerTurn } = unit;
     const applicable = applicablePerTurn[0];
     const base = {
@@ -235,6 +469,10 @@ async function gradeUnit(unit, cellResult, judge) {
         // pre-BATCH-21 cell's `<id>.json` keeps its exact keys (JSON.stringify drops undefined).
         toolResults: cellResult.toolResults,
         durationMs: cellResult.durationMs,
+        // BATCH-25 — omitted entirely (not set to `undefined`) for a non-classifier suite, so a
+        // pre-BATCH-25 `<id>.json` keeps its exact key set.
+        ...(classification !== undefined ? { classification } : {}),
+        ...(caseTags(unit.evalCase).length > 0 ? { tags: caseTags(unit.evalCase) } : {}),
     };
     if (!cellResult.ok) {
         return {
@@ -258,7 +496,8 @@ async function gradeUnit(unit, cellResult, judge) {
     const answer = cellResult.answer ?? '';
     const tools = cellResult.tools ?? [];
     const toolResults = cellResult.toolResults ?? [];
-    const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge);
+    const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge, classification);
+    reasons.push(...gradeModelFree(evalCase, classification));
     // A cell PASSES iff nothing was recorded against it: every applicable block's deterministic
     // checks, tool-trace checks, and judge all passed (a passing judge appends no reason). For a
     // single applicable block this is exactly the pre-BATCH-12 verdict.
@@ -286,7 +525,7 @@ async function gradeUnit(unit, cellResult, judge) {
  * `sutOk:false` (exit-2-eligible, like a single-shot SUT failure) while a turn-1-ran/turn-2-failed
  * cell is `sutOk:true`, a real product signal → exit 1.
  */
-async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
+async function gradeConversationUnit(unit, outcomes, cellResult, judge, classifications) {
     const { evalCase, identity, applicablePerTurn } = unit;
     const turnOutcomes = outcomes ?? [];
     const turnResults = [];
@@ -303,6 +542,7 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
                 'which is a suite-authoring error.');
         }
         const outcome = turnOutcomes[i];
+        const classification = classifications?.[i];
         const label = `turn ${i + 1}`;
         if (!outcome || !outcome.ok) {
             // This turn produced no answer: the conversation errored on it, or aborted before reaching it.
@@ -325,6 +565,7 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
                 ok: false,
                 verdict: 'FAIL',
                 reasons: [detail],
+                ...(classification !== undefined ? { classification } : {}),
             });
             cellReasons.push(`${label}: ${detail}`);
             continue;
@@ -333,7 +574,8 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
         const answer = outcome.answer ?? '';
         const tools = outcome.tools ?? [];
         const toolResults = outcome.toolResults ?? [];
-        const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge);
+        const { reasons, deterministicFailures, judgeOutcome } = await gradeApplicableBlocks(answer, tools, toolResults, applicable, evalCase.passThreshold, judge, classification);
+        reasons.push(...gradeModelFree(evalCase, classification));
         const turnVerdict = reasons.length === 0 ? 'PASS' : 'FAIL';
         turnResults.push({
             user: turn.user,
@@ -347,6 +589,7 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
             checks: { passed: deterministicFailures.length === 0, failures: deterministicFailures },
             judge: judgeOutcome,
             reasons,
+            ...(classification !== undefined ? { classification } : {}),
         });
         for (const reason of reasons)
             cellReasons.push(`${label}: ${reason}`);
@@ -362,6 +605,13 @@ async function gradeConversationUnit(unit, outcomes, cellResult, judge) {
         sutOk: anyTurnRan,
         reasons: cellReasons,
         turns: turnResults,
+        // BATCH-25 — the cell-level classification is the LAST round's: a negotiation corpus asserts its
+        // outcome on where the exchange ENDED (reject · reject · escalate), so that is the value the
+        // confusion matrix and the metrics must read. Every round's own value is in `turns`.
+        ...(classifications !== undefined && classifications.length > 0
+            ? { classification: classifications[classifications.length - 1] }
+            : {}),
+        ...(caseTags(evalCase).length > 0 ? { tags: caseTags(evalCase) } : {}),
     };
 }
 //# sourceMappingURL=evalRunner.js.map

package/dist/evalRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAwE/D;;uEAEuE;AACvE,SAAS,oBAAoB,CAAC,WAA4B,EAAE,QAA4B;IACtF,IAAI,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,OAAO,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE;gBAC5E,QAAQ;gBACR,QAAQ;gBACR,yFAAyF;gBACzF,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CACnE;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,kGAAkG;IAClG,4FAA4F;IAC5F,gGAAgG;IAChG,gGAAgG;IAChG,4FAA4F;IAC5F,gGAAgG;IAChG,kGAAkG;IAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5E,gGAAgG;IAChG,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,UAAU,GAAG,CAAC,QAA4B,EAAa,EAAE;QAC7D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,gGAAgG;IAChG,yFAAyF;IACzF,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,CAAC,QAA4B,EAAqB,EAAE;QAC7E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,eAAe,CAAC;QACjC,CAAC;QACD,MAAM,uBAAuB,GAAG,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2DAA2D,QAAQ,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,8FAA8F;QAC9F,0FAA0F;QAC1F,8BAA8B;QAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,4FAA4F;IAC5F,kGAAkG;IAClG,gGAAgG;IAChG,qFAAqF;IACrF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,MAAM,eAAe,GAAc,KAAK,EAAE,IAAI,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;QACpD,2FAA2F;QAC3F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,2FAA2F;QAC3F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7C,CAAC;QACF,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACzD;wGACgG;QAChG,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,MAAM,qBAAqB,CACzB,IAAI,EACJ,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAC/C,UAAU,EACV,OAAO,CAAC,KAAK,CACd,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5F,CAAC;AAKD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,iFAAiF;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yFAAyF;IACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qBAAqB;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAe,EACf,WAA+B,EAC/B,UAA6B,EAC7B,aAAqB,EACrB,KAA0B;IAM1B,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,6FAA6F;QAC7F,6FAA6F;QAC7F,iEAAiE;QACjE,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE,qBAAqB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,YAAY,EAAE,GAAG,kBAAkB,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,OAAqB,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACjD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;oBAChD,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,OAAO,CAAC,IAAI,uBAAuB,aAAa,EAAE;wBACtE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,GAAG,OAAO,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,8FAA8F;QAC9F,2EAA2E;QAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,4FAA4F;QAC5F,yFAAyF;QACzF,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,UAAU,EAAE,UAAU,CAAC,UAAU;KAClC,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,6FAA6F;IAC7F,mGAAmG;IACnG,2FAA2F;IAC3F,gGAAgG;IAChG,qGAAqG;IACrG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;YACpE,IAAI,QAAQ,IAAI,WAAW,8DAA8D;YACzF,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,EAAE,CAAC;IACjD,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;IAEF,4FAA4F;IAC5F,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,OAAO,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,qBAAqB,CAClC,IAAc,EACd,QAAsC,EACtC,UAAsB,EACtB,KAA0B;IAE1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,IAAI,EAAE,CAAC;IAEpC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAExC,8FAA8F;QAC9F,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;gBACpE,IAAI,QAAQ,IAAI,WAAW,WAAW,CAAC,iDAAiD;gBACxF,mCAAmC,CACtC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAC5B,+FAA+F;YAC/F,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,EAAE,KAAK;gBAAE,MAAM,GAAG,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC;iBAC3D,IAAI,OAAO;gBAAE,MAAM,GAAG,iBAAiB,CAAC;iBACxC,IAAI,QAAQ,KAAK,SAAS;gBAC7B,MAAM,GAAG,mBAAmB,UAAU,CAAC,KAAK,IAAI,iCAAiC,EAAE,CAAC;;gBACjF,MAAM,GAAG,0DAA0D,CAAC;YAEzE,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,YAAY,EAAE,OAAO,EAAE,YAAY;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK;gBACrB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;QAC9C,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,CACN,CAAC;QACF,MAAM,WAAW,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;YACvF,KAAK,EAAE,YAAY;YACnB,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iGAAiG;QACjG,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO;QACP,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;KACnB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evalRunner.js","sourceRoot":"","sources":["../src/evalRunner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAgB/D,OAAO,EAAE,8BAA8B,EAAE,MAAM,mBAAmB,CAAC;AAMnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAAE,0BAA0B,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AA0EzE;;;;;;;;GAQG;AACH,SAAS,QAAQ,CAAC,QAAkB;IAClC,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED;;uEAEuE;AACvE,SAAS,oBAAoB,CAAC,WAA4B,EAAE,QAA4B;IACtF,IAAI,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChF,OAAO,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAgB;IAClC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE;gBAC5E,QAAQ;gBACR,QAAQ;gBACR,yFAAyF;gBACzF,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CACnE;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAgB,EAChB,OAA4B;IAE5B,gGAAgG;IAChG,iGAAiG;IACjG,kGAAkG;IAClG,gGAAgG;IAChG,mFAAmF;IACnF,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CACb,uFAAuF;YACrF,oFAAoF;YACpF,oFAAoF,CACvF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAChC,kGAAkG;IAClG,4FAA4F;IAC5F,gGAAgG;IAChG,gGAAgG;IAChG,4FAA4F;IAC5F,gGAAgG;IAChG,kGAAkG;IAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5E,gGAAgG;IAChG,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,UAAU,GAAG,CAAC,QAA4B,EAAa,EAAE;QAC7D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC;QACzB,CAAC;QACD,MAAM,eAAe,GAAG,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjE,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC,CAAC;IAEF,gGAAgG;IAChG,yFAAyF;IACzF,+FAA+F;IAC/F,MAAM,kBAAkB,GAAG,CAAC,QAA4B,EAAqB,EAAE;QAC7E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,eAAe,CAAC;QACjC,CAAC;QACD,MAAM,uBAAuB,GAAG,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2DAA2D,QAAQ,IAAI,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,IAAI,CAAC,MAAM;QACf,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,KAAK;QACjB,8FAA8F;QAC9F,0FAA0F;QAC1F,8BAA8B;QAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KACrC,CAAC,CAAC,CAAC;IAEJ,4FAA4F;IAC5F,kGAAkG;IAClG,gGAAgG;IAChG,qFAAqF;IACrF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,mGAAmG;IACnG,0FAA0F;IAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvD,gGAAgG;IAChG,+EAA+E;IAC/E,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA6B,CAAC;IAE9D,MAAM,eAAe,GAAc,KAAK,EAAE,IAAI,EAAE,EAAE;QAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;QAEpD,6FAA6F;QAC7F,+FAA+F;QAC/F,iFAAiF;QACjF,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC;gBAChC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpD,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC7B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;gBAClC,uFAAuF;gBACvF,2FAA2F;gBAC3F,EAAE;gBACF,wFAAwF;gBACxF,0FAA0F;gBAC1F,sFAAsF;gBACtF,uFAAuF;gBACvF,mFAAmF;gBACnF,4FAA4F;gBAC5F,sEAAsE;gBACtE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAC/B,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,8BAA8B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC/E,EAAE,QAAQ;oBACZ,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,QAAQ,CAC5E;aACF,CAAC,CAAC;YACH,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YAChD,oBAAoB,CAAC,GAAG,CACtB,IAAI,CAAC,UAAU,EACf,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACzB,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,MAAM,EAAE,OAAO,CAAC,SAAS;gBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC,CACJ,CAAC;YACF,OAAO;gBACL,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS;gBAC9B,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK;aAC1B,CAAC;QACJ,CAAC;QAED,2FAA2F;QAC3F,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,2FAA2F;QAC3F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7C,CAAC;QACF,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACzD;wGACgG;QAChG,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACjE,iGAAiG;QACjG,6EAA6E;QAC7E,MAAM,cAAc,GAClB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC;YAC7B,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;YACrB,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;QACvE,MAAM,eAAe,GAAG,0BAA0B,CAChD,IAAI,EACJ,IAAI,EACJ,cAAc,EACd,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAC5C,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CACV,MAAM,qBAAqB,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,CACxF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,OAAO,GAAqB;QAChC,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,KAAK,EAAE,OAAO;KACf,CAAC;IAEF,mGAAmG;IACnG,uFAAuF;IACvF,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,CAAC,cAAc,GAAG,yBAAyB,CAChD,IAAI;QACJ,yFAAyF;QACzF,wBAAwB;QACxB,KAAK,CAAC,OAAO,IAAI,EAAE,EACnB,iBAAiB,CAAC,OAAO,CAAC,CAC3B,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAyB;IACzD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9B,EAAE,EAAE,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE;QAClF,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE;QACvB,aAAa,EAAE,MAAM,CAAC,cAAc,EAAE,aAAa;QACnD,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,cAAc;QACrD,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,WAAW;QAC/C,YAAY,EAAE,MAAM,CAAC,cAAc,EAAE,YAAY;QACjD,MAAM,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS;KAC5D,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,0BAA0B,CACjC,IAAc,EACd,IAAwC,EACxC,cAAsC,EACtC,gBAA+C;IAE/C,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvD,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE,WAAW,CAAC;QAC/F,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CACpC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,CAC5C,EAAE,YAAY,CAAC;QAEhB,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAErC,IAAI,WAA+B,CAAC;QACpC,IAAI,YAAgC,CAAC;QACrC,IAAI,GAAuB,CAAC;QAE5B,IAAI,OAAO,EAAE,CAAC;YACZ,6FAA6F;YAC7F,wFAAwF;YACxF,0CAA0C;YAC1C,WAAW,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxF,YAAY;gBACV,OAAO,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;oBACnC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;oBACpD,CAAC,CAAC,SAAS,CAAC;YAChB,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;QAC1B,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,WAAW,GAAG,0BAA0B,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9E,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,YAAY,GAAG,0BAA0B,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACnF,CAAC;YACD,wFAAwF;YACxF,oFAAoF;YACpF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAClD,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,MAAM;gBAAE,GAAG,GAAG,SAAS,CAAC;QACvE,CAAC;QAED,OAAO;YACL,aAAa;YACb,cAAc;YACd,WAAW;YACX,YAAY;YACZ,GAAG;YACH,UAAU,EAAE,OAAO,EAAE,UAAU;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;iCAEiC;AACjC,SAAS,oBAAoB,CAAC,KAAyB,EAAE,QAAkB;IACzE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC;AAC/D,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,wBAAwB,CAC/B,KAAsB,EACtB,cAAkD;IAElD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,cAAc,EAAE,WAAW,CAAC;QAC3C,IAAI,MAAM,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,WAAW,cAAc,MAAM,IAAI,QAAQ,GAAG,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,cAAc,EAAE,YAAY,CAAC;QAC5C,IAAI,MAAM,KAAK,KAAK,CAAC,YAAY,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,YAAY,cAAc,MAAM,IAAI,QAAQ,GAAG,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;4EAI4E;AAC5E,SAAS,cAAc,CACrB,QAAkB,EAClB,cAAkD;IAElD,IAAI,CAAC,QAAQ,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,cAAc,EAAE,UAAU,CAAC;IACzC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO;YACL,0FAA0F;gBACxF,WAAW;SACd,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAKD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAyB;IACxD,iFAAiF;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yFAAyF;IACzF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,2FAA2F;IAC3F,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,qBAAqB;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAc,EACd,KAAe,EACf,WAA+B,EAC/B,UAA6B,EAC7B,aAAqB,EACrB,KAA0B,EAC1B,cAAkD;IAMlD,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,YAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,yFAAyF;QACzF,6FAA6F;QAC7F,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,6FAA6F;QAC7F,6FAA6F;QAC7F,iEAAiE;QACjE,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACnE,0FAA0F;QAC1F,2FAA2F;QAC3F,+FAA+F;QAC/F,aAAa;QACb,MAAM,sBAAsB,GAAG,wBAAwB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAC/E,qBAAqB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CACV,GAAG,MAAM,CAAC,QAAQ,EAClB,GAAG,YAAY,EACf,GAAG,kBAAkB,EACrB,GAAG,sBAAsB,CAC1B,CAAC;QAEF,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,OAAqB,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;gBACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;gBACjD,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpC,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;oBAChD,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,OAAO,CAAC,IAAI,uBAAuB,aAAa,EAAE;wBACtE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,IAAI,YAAY,KAAK,SAAS;gBAAE,YAAY,GAAG,OAAO,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,UAAsB,EACtB,KAA0B,EAC1B,cAAkD;IAElD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG;QACX,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,8FAA8F;QAC9F,2EAA2E;QAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,4FAA4F;QAC5F,yFAAyF;QACzF,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,wFAAwF;QACxF,oDAAoD;QACpD,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACjF,CAAC;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,6FAA6F;IAC7F,mGAAmG;IACnG,2FAA2F;IAC3F,gGAAgG;IAChG,qGAAqG;IACrG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;YACpE,IAAI,QAAQ,IAAI,WAAW,8DAA8D;YACzF,wBAAwB,CAC3B,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,EAAE,CAAC;IACjD,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,EACL,cAAc,CACf,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,OAAO,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO;QACL,GAAG,IAAI;QACP,OAAO;QACP,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;QACvF,KAAK,EAAE,YAAY;QACnB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,qBAAqB,CAClC,IAAc,EACd,QAAsC,EACtC,UAAsB,EACtB,KAA0B,EAC1B,eAAqD;IAErD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,IAAI,EAAE,CAAC;IAEpC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAExC,8FAA8F;QAC9F,iGAAiG;QACjG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,MAAM,cAAc;gBACpE,IAAI,QAAQ,IAAI,WAAW,WAAW,CAAC,iDAAiD;gBACxF,mCAAmC,CACtC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAC5B,+FAA+F;YAC/F,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,EAAE,KAAK;gBAAE,MAAM,GAAG,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC;iBAC3D,IAAI,OAAO;gBAAE,MAAM,GAAG,iBAAiB,CAAC;iBACxC,IAAI,QAAQ,KAAK,SAAS;gBAC7B,MAAM,GAAG,mBAAmB,UAAU,CAAC,KAAK,IAAI,iCAAiC,EAAE,CAAC;;gBACjF,MAAM,GAAG,0DAA0D,CAAC;YAEzE,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,YAAY,EAAE,OAAO,EAAE,YAAY;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK;gBACrB,WAAW,EAAE,OAAO,EAAE,WAAW;gBACjC,EAAE,EAAE,KAAK;gBACT,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5D,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;QAC9C,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,GAAG,MAAM,qBAAqB,CAClF,MAAM,EACN,KAAK,EACL,WAAW,EACX,UAAU,EACV,QAAQ,CAAC,aAAa,EACtB,KAAK,EACL,cAAc,CACf,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,EAAE,EAAE,IAAI;YACR,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,QAAQ,EAAE,qBAAqB,EAAE;YACvF,KAAK,EAAE,YAAY;YACnB,OAAO;YACP,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAoB,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iGAAiG;QACjG,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,OAAO;QACP,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;QAClB,iGAAiG;QACjG,6FAA6F;QAC7F,qFAAqF;QACrF,GAAG,CAAC,eAAe,KAAK,SAAS,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;YAC7D,CAAC,CAAC,EAAE,cAAc,EAAE,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;YACjE,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC;AACJ,CAAC"}

package/dist/evalSuite.d.ts

@@ -9,8 +9,8 @@ import type { EvalSuite } from '#src/evalTypes.js';
  * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
  * - Malformed YAML.
  * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
- * - `target.type` other than `"gth-agent"`, `"adk-agent"`, or `"ag-ui"` — other pluggable CLI/HTTP
- *   targets are out of scope.
+ * - `target.type` other than `"gth-agent"`, `"adk-agent"`, `"ag-ui"`, or `"rater"` — other pluggable
+ *   CLI/HTTP targets are out of scope.
  * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
  *   switch is the `--identities` direction, replaced by the suite-level `identities` list.
  * - A `"adk-agent"` (BATCH-14) target missing its `url`; an `adk-agent` suite that ALSO uses the
@@ -41,6 +41,16 @@ import type { EvalSuite } from '#src/evalTypes.js';
  * - A tool-RESULT assertion (`must_error` / `tool_result_json_path`, BATCH-21) against an
  *   `adk-agent` OR `ag-ui` target — tool results exist only on the in-process `gth-agent` target
  *   (the AG-UI wire streams call names but no result payloads; A2A exposes no tool trace at all).
+ * - A `"rater"` (BATCH-25 Half B) target missing its `rung`, naming one that is not on the approvals
+ *   ladder, or carrying a `profile`; a `rater` suite with no `classification:` block (the classifier
+ *   layer would be inert and every case would silently run through the ordinary agent); a `rater`
+ *   suite using the `identities` matrix (the classification seam is per-case, not per-identity); or
+ *   a `rater` suite using ANY tool assertion (no agent runs, so there is no trace to grade).
+ * - `model_free: true` on a case of any target EXCEPT `rater` — running a case through an agent IS
+ *   a model call, so the flag could never bite; and `model_free: true` on a case that ALSO declares
+ *   a `judge:` rubric, since the judge is a second model call the target's `modelCalls` cannot see.
+ * - `forced_by` on any target except `rater`, or naming something that is not a mechanism of the
+ *   approvals gate.
  * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.