@gaunt-sloth/batch 2.0.0-beta.10 → 2.0.0-beta.11

package/dist/evalCompare.d.ts

@@ -71,6 +71,101 @@ export interface RunDiffEntry {
     before: string;
     after: string;
 }
+/**
+ * BATCH-33 — how a run-over-run JUDGE-SCORE drift report is filtered.
+ *
+ * ## Why there is a filter at all, and why the raw form is not a member of this union
+ *
+ * A judge score is a model's 0-10 opinion, and it wobbles between identical runs. Reporting every
+ * per-case delta therefore prints a section on every run, most of it noise, and trains the reader to
+ * skip it — at which point the real slide is skipped along with it. That is strictly WORSE than
+ * reporting no drift at all: a section nobody reads still spends the run's credibility, which is the
+ * same failure BATCH-25's untrusted metric ran into.
+ *
+ * So the raw unfiltered form is deliberately not representable. There is no `all` member, and no
+ * value of the knobs below reaches one — {@link parseJudgeDriftFilter} rejects `min:0` and bounds
+ * the threshold-ward tolerance, because a knob that admits its own degenerate value ships the form
+ * this type exists to withhold.
+ *
+ * ## What each member is for
+ *
+ * - `threshold-ward` (**the default**) — report a case only when its score CROSSED the pass
+ *   threshold, or moved DOWN to within `tolerance` points of it. Distance to the gate is the thing
+ *   worth alerting on: a 10 → 8 that stays clear of a gate at 6 is a different event from a 7 → 6
+ *   sitting on it, and only the second is about to cost a verdict. It is the default because it is
+ *   the one filter whose false-positive rate on a stable suite is near zero: at the default
+ *   tolerance a score must land at or below its own gate to report, so a corpus grading anywhere
+ *   above its gate wobbles freely and prints nothing.
+ * - `min-points` — report any movement of at least N points, either direction. The blunt
+ *   instrument: it sees large swings wherever they land, and pays for that by firing on a wobbly
+ *   judge no matter how much headroom the case had.
+ * - `mean` — report only the suite-level mean. Averaging over the corpus cancels the wobble, so it
+ *   is the quietest form of all and the one that cannot say WHICH case moved.
+ * - `off` — no drift report. Opting out is not the raw form; it prints less, not more.
+ */
+export type JudgeDriftFilter = {
+    mode: 'threshold-ward';
+    tolerance: number;
+} | {
+    mode: 'min-points';
+    points: number;
+} | {
+    mode: 'mean';
+} | {
+    mode: 'off';
+};
+/**
+ * How many points above the gate still counts as sitting ON it, when the caller names no tolerance.
+ *
+ * Zero, and the reason is a measurement rather than a preference. A local judge asked to re-rate one
+ * fixed answer against one fixed rubric twelve times returned rates spanning a full point (7 and 8,
+ * on the same input every time). So one point of movement carries no information: it is the width of
+ * the noise. A shoulder of 1 would report every cell that wobbled from 8 to 7 against the default
+ * gate of 6 — an ordinary judged suite grades right there, so the section would fire on a stable
+ * re-run, which is the one failure this node exists to avoid.
+ *
+ * At zero the rule still reports both movements the node names: a 7 → 6 lands ON the gate and
+ * reports, a 10 → 8 stays clear and does not. The shoulder was buying nothing those two clauses did
+ * not already cover, and was costing exactly the measured noise band. Widen it deliberately with
+ * `--drift threshold-ward:<n>` on a corpus whose judge is steadier than this.
+ */
+export declare const DEFAULT_JUDGE_DRIFT_TOLERANCE = 0;
+/**
+ * The widest tolerance a caller may ask for. The judge scale is 0-10 and a typical gate is 6, so a
+ * shoulder much wider than this covers the whole usable band and every downward move reports —
+ * the unfiltered form by another route, which is what the bound exists to refuse.
+ */
+export declare const MAX_JUDGE_DRIFT_TOLERANCE = 3;
+/** The filter {@link diffRuns} applies when the caller names none. */
+export declare const DEFAULT_JUDGE_DRIFT_FILTER: JudgeDriftFilter;
+/**
+ * Parse a `--drift` spec into a {@link JudgeDriftFilter}, throwing on anything unrecognised —
+ * including the degenerate values that would reconstitute the raw per-case report.
+ *
+ * Accepted: `threshold-ward` (or `threshold-ward:<tolerance>`), `min:<points>`, `mean`, `off`.
+ */
+export declare function parseJudgeDriftFilter(spec: string): JudgeDriftFilter;
+/** One case whose JUDGE SCORE moved between two runs, as kept by a {@link JudgeDriftFilter}. */
+export interface JudgeDriftEntry {
+    id: string;
+    before: number;
+    after: number;
+    /** `after - before`. Negative is a slide toward the gate. */
+    delta: number;
+    /** The gate the movement is measured against — THIS run's, since that is the one now in force. */
+    passThreshold: number;
+    /** Why the filter kept it: the score crossed the gate, slid onto the gate's shoulder, or simply
+     * moved far enough for a `min-points` filter. */
+    reason: 'crossed' | 'near-threshold' | 'moved';
+}
+/** The suite-level judge-score mean — the `mean` filter's whole output. */
+export interface JudgeDriftMean {
+    before: number;
+    after: number;
+    delta: number;
+    /** Cases carrying a rate on BOTH sides, i.e. the mean's denominator. */
+    cases: number;
+}
 /** The run-over-run diff. */
 export interface RunDiff {
     /** Cases in both runs. */
@@ -88,6 +183,20 @@ export interface RunDiff {
         after: number | null;
         delta: number | null;
     }[];
+    /**
+     * BATCH-33 — per-case judge-score movements the active filter kept. The signal a VERDICT-only
+     * diff structurally cannot see: a judge-graded suite with no `classification:` block has neither
+     * `reclassified` nor `metricDeltas`, so before this its diff was strictly binary and a score
+     * sliding toward its gate reported "no change." until the run it finally broke.
+     *
+     * EMPTY under the `mean` and `off` filters, which report no per-case rows at all.
+     */
+    judgeDrift: JudgeDriftEntry[];
+    /** The suite-level mean, under the `mean` filter only. */
+    judgeDriftMean?: JudgeDriftMean;
+    /** The filter that produced the two fields above, echoed so the render can name it: a quiet drift
+     * section means nothing unless the reader can see WHICH filter was quiet. */
+    judgeDriftFilter: JudgeDriftFilter;
     /** Ids in only one of the two runs — reported, so a shrunken corpus cannot read as "no change". */
     onlyInBefore: string[];
     onlyInAfter: string[];
@@ -100,8 +209,15 @@ export interface RunDiff {
  * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
  * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
  * and that is exactly the drift a pass-rate comparison cannot see.
+ *
+ * A fourth, {@link RunDiff.judgeDrift}, covers the suites the other three cannot: `reclassified`
+ * and `metricDeltas` both need a `classification:` block, so an ordinary judge-graded corpus got a
+ * strictly binary diff and learned nothing until a score finally broke its gate. `filter` decides
+ * how much of that movement is worth printing — see {@link JudgeDriftFilter} for why it defaults to
+ * the threshold-ward one and why the unfiltered form is not on offer. **The default lives here, not
+ * only in the CLI**, so every caller gets the quiet behaviour without having to know to ask.
  */
-export declare function diffRuns(before: EvalSuiteSummary, after: EvalSuiteSummary): RunDiff;
+export declare function diffRuns(before: EvalSuiteSummary, after: EvalSuiteSummary, filter?: JudgeDriftFilter): RunDiff;
 /** Render a {@link RunDiff} as plain lines. */
 export declare function renderRunDiff(diff: RunDiff): string[];
 /** Does this suite declare a sweep? Small helper so the command reads declaratively. */

package/dist/evalCompare.js

@@ -128,6 +128,137 @@ function cellMetricValue(column, metricName, tag) {
         return 'n/a';
     return formatTally(tally);
 }
+/**
+ * How many points above the gate still counts as sitting ON it, when the caller names no tolerance.
+ *
+ * Zero, and the reason is a measurement rather than a preference. A local judge asked to re-rate one
+ * fixed answer against one fixed rubric twelve times returned rates spanning a full point (7 and 8,
+ * on the same input every time). So one point of movement carries no information: it is the width of
+ * the noise. A shoulder of 1 would report every cell that wobbled from 8 to 7 against the default
+ * gate of 6 — an ordinary judged suite grades right there, so the section would fire on a stable
+ * re-run, which is the one failure this node exists to avoid.
+ *
+ * At zero the rule still reports both movements the node names: a 7 → 6 lands ON the gate and
+ * reports, a 10 → 8 stays clear and does not. The shoulder was buying nothing those two clauses did
+ * not already cover, and was costing exactly the measured noise band. Widen it deliberately with
+ * `--drift threshold-ward:<n>` on a corpus whose judge is steadier than this.
+ */
+export const DEFAULT_JUDGE_DRIFT_TOLERANCE = 0;
+/**
+ * The widest tolerance a caller may ask for. The judge scale is 0-10 and a typical gate is 6, so a
+ * shoulder much wider than this covers the whole usable band and every downward move reports —
+ * the unfiltered form by another route, which is what the bound exists to refuse.
+ */
+export const MAX_JUDGE_DRIFT_TOLERANCE = 3;
+/** The filter {@link diffRuns} applies when the caller names none. */
+export const DEFAULT_JUDGE_DRIFT_FILTER = {
+    mode: 'threshold-ward',
+    tolerance: DEFAULT_JUDGE_DRIFT_TOLERANCE,
+};
+/**
+ * Parse a `--drift` spec into a {@link JudgeDriftFilter}, throwing on anything unrecognised —
+ * including the degenerate values that would reconstitute the raw per-case report.
+ *
+ * Accepted: `threshold-ward` (or `threshold-ward:<tolerance>`), `min:<points>`, `mean`, `off`.
+ */
+export function parseJudgeDriftFilter(spec) {
+    const parts = spec.trim().split(':');
+    if (parts.length > 2)
+        throw new Error(driftSpecError(spec));
+    const mode = parts[0].trim().toLowerCase();
+    const value = parts[1];
+    if (mode === 'off' || mode === 'mean') {
+        if (value !== undefined)
+            throw new Error(driftSpecError(spec));
+        return mode === 'off' ? { mode: 'off' } : { mode: 'mean' };
+    }
+    if (mode === 'min') {
+        const points = driftNumber(value, spec);
+        // 0 or less would keep every movement — the unfiltered report this filter exists to avoid.
+        if (points < 1 || points > 10) {
+            throw new Error(`--drift min:<points> takes 1-10, so "${spec}" is not accepted: it would report every ` +
+                'judge-score movement, which is the unfiltered form that trains readers to skip the ' +
+                'section.');
+        }
+        return { mode: 'min-points', points };
+    }
+    if (mode === 'threshold-ward' || mode === 'threshold') {
+        if (value === undefined)
+            return { mode: 'threshold-ward', tolerance: DEFAULT_JUDGE_DRIFT_TOLERANCE };
+        const tolerance = driftNumber(value, spec);
+        if (tolerance < 0 || tolerance > MAX_JUDGE_DRIFT_TOLERANCE) {
+            throw new Error(`--drift threshold-ward:<tolerance> takes 0-${MAX_JUDGE_DRIFT_TOLERANCE}, so "${spec}" is ` +
+                'not accepted: a shoulder that wide reports nearly every downward move, which is the ' +
+                'unfiltered form that trains readers to skip the section.');
+        }
+        return { mode: 'threshold-ward', tolerance };
+    }
+    throw new Error(driftSpecError(spec));
+}
+function driftSpecError(spec) {
+    return (`unrecognised --drift filter "${spec}". Use "threshold-ward" (the default, optionally ` +
+        `"threshold-ward:<0-${MAX_JUDGE_DRIFT_TOLERANCE}>"), "min:<1-10>", "mean", or "off".`);
+}
+function driftNumber(raw, spec) {
+    if (raw === undefined || raw.trim() === '')
+        throw new Error(driftSpecError(spec));
+    const value = Number(raw.trim());
+    if (!Number.isInteger(value))
+        throw new Error(driftSpecError(spec));
+    return value;
+}
+/**
+ * The judge rate representing one CELL, or `undefined` when no verdict was rendered for it.
+ *
+ * A single-turn cell carries its own {@link EvalCaseResult.judge}. A MULTI-TURN cell leaves that
+ * unset and grades per turn, so its representative rate is the LOWEST any turn scored: the cell
+ * passes iff every turn passes, which makes the weakest turn the one sitting nearest the gate and
+ * therefore the one this report is about. Reading only the top-level field would hand every
+ * multi-turn suite an empty drift section forever — the same silence this node exists to end.
+ */
+function judgeRateOf(result) {
+    const direct = rateOfOutcome(result.judge);
+    if (direct !== undefined)
+        return direct;
+    const turnRates = (result.turns ?? [])
+        .map((turn) => rateOfOutcome(turn.judge))
+        .filter((rate) => rate !== undefined);
+    return turnRates.length === 0 ? undefined : Math.min(...turnRates);
+}
+/** A usable 0-10 rate, or `undefined`. A judge that errored, timed out, returned something
+ * unparseable or was never asked has no score to compare, and substituting one would report a slide
+ * nobody measured. */
+function rateOfOutcome(judge) {
+    if (judge?.ok !== true)
+        return undefined;
+    const rate = judge.verdict?.rate;
+    return typeof rate === 'number' && Number.isFinite(rate) ? rate : undefined;
+}
+/**
+ * Does one movement survive the filter? Returns the reason it was kept, or `undefined`.
+ *
+ * The threshold-ward rule is two clauses and BOTH are load-bearing:
+ *
+ * - a CROSSING of the gate, either direction, always reports. That is the moment the score changed
+ *   what it is worth, and its numbers are what explain the verdict flip printed beside it.
+ * - otherwise, only a DOWNWARD move landing within `tolerance` of the gate. Direction alone is not
+ *   the rule — a 10 → 8 is downward and still clear of a gate at 6, and keeping it is exactly how
+ *   the section fills with wobble. Proximity alone is not the rule either, or a case recovering
+ *   6 → 7 would report. It is the pair, and a reader tempted to simplify one clause away should
+ *   expect the section to go noisy on the next stable re-run.
+ */
+function driftReason(before, after, passThreshold, filter) {
+    if (filter.mode === 'min-points') {
+        return Math.abs(after - before) >= filter.points ? 'moved' : undefined;
+    }
+    if (filter.mode !== 'threshold-ward')
+        return undefined;
+    if (before >= passThreshold !== after >= passThreshold)
+        return 'crossed';
+    if (after < before && after <= passThreshold + filter.tolerance)
+        return 'near-threshold';
+    return undefined;
+}
 /** The key a cell is diffed on across runs: id plus identity, since a matrix cell's id alone is
  * ambiguous. */
 function diffKey(result) {
@@ -140,19 +271,44 @@ function diffKey(result) {
  * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
  * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
  * and that is exactly the drift a pass-rate comparison cannot see.
+ *
+ * A fourth, {@link RunDiff.judgeDrift}, covers the suites the other three cannot: `reclassified`
+ * and `metricDeltas` both need a `classification:` block, so an ordinary judge-graded corpus got a
+ * strictly binary diff and learned nothing until a score finally broke its gate. `filter` decides
+ * how much of that movement is worth printing — see {@link JudgeDriftFilter} for why it defaults to
+ * the threshold-ward one and why the unfiltered form is not on offer. **The default lives here, not
+ * only in the CLI**, so every caller gets the quiet behaviour without having to know to ask.
  */
-export function diffRuns(before, after) {
+export function diffRuns(before, after, filter = DEFAULT_JUDGE_DRIFT_FILTER) {
     const beforeByKey = new Map(before.cases.map((result) => [diffKey(result), result]));
     const afterByKey = new Map(after.cases.map((result) => [diffKey(result), result]));
     const regressed = [];
     const fixed = [];
     const reclassified = [];
+    const ratePairs = [];
+    let lostRates = 0;
+    let movedGates = 0;
     let compared = 0;
     for (const [key, afterCase] of afterByKey) {
         const beforeCase = beforeByKey.get(key);
         if (!beforeCase)
             continue;
         compared += 1;
+        const beforeRate = judgeRateOf(beforeCase);
+        const afterRate = judgeRateOf(afterCase);
+        if (beforeRate !== undefined && afterRate !== undefined) {
+            ratePairs.push({
+                key,
+                before: beforeRate,
+                after: afterRate,
+                passThreshold: afterCase.passThreshold,
+            });
+            if (beforeCase.passThreshold !== afterCase.passThreshold)
+                movedGates += 1;
+        }
+        else if (beforeRate !== undefined) {
+            lostRates += 1;
+        }
         if (beforeCase.verdict === 'PASS' && afterCase.verdict === 'FAIL') {
             regressed.push({ id: key, before: 'PASS', after: 'FAIL' });
         }
@@ -181,7 +337,48 @@ export function diffRuns(before, after) {
             delta: beforeValue === null || afterValue === null ? null : afterValue - beforeValue,
         });
     }
+    // BATCH-33 — the judge-score drift, as much of it as the filter keeps. `mean` reports only the
+    // suite aggregate (no per-case rows); every other filter reports rows and no aggregate.
+    const judgeDrift = [];
+    let judgeDriftMean;
+    if (filter.mode === 'mean') {
+        if (ratePairs.length > 0) {
+            const mean = (pick) => ratePairs.reduce((sum, pair) => sum + pick(pair), 0) / ratePairs.length;
+            const meanBefore = mean((pair) => pair.before);
+            const meanAfter = mean((pair) => pair.after);
+            judgeDriftMean = {
+                before: meanBefore,
+                after: meanAfter,
+                delta: meanAfter - meanBefore,
+                cases: ratePairs.length,
+            };
+        }
+    }
+    else {
+        for (const pair of ratePairs) {
+            const reason = driftReason(pair.before, pair.after, pair.passThreshold, filter);
+            if (reason === undefined)
+                continue;
+            judgeDrift.push({
+                id: pair.key,
+                before: pair.before,
+                after: pair.after,
+                delta: pair.after - pair.before,
+                passThreshold: pair.passThreshold,
+                reason,
+            });
+        }
+    }
     const warnings = [];
+    if (lostRates > 0 && filter.mode !== 'off') {
+        warnings.push(`${lostRates} case(s) were judged in the baseline and produced no judge score in this run, ` +
+            'so their drift could not be computed. A quiet drift section here is not "the scores held".');
+    }
+    if (movedGates > 0 && filter.mode !== 'off') {
+        warnings.push(`${movedGates} case(s) changed their pass threshold between the two runs. Drift is measured ` +
+            "against THIS run's gate, so some of the distance reported here is the gate moving rather " +
+            'than the score.');
+    }
     if (onlyInBefore.length > 0 || onlyInAfter.length > 0) {
         warnings.push(`the two runs do not cover the same cases: ${onlyInBefore.length} only in the baseline, ` +
             `${onlyInAfter.length} only in this run. The comparison covers ${compared} case(s); a ` +
@@ -193,6 +390,9 @@ export function diffRuns(before, after) {
         fixed,
         reclassified,
         metricDeltas,
+        judgeDrift,
+        judgeDriftMean,
+        judgeDriftFilter: filter,
         onlyInBefore,
         onlyInAfter,
         warnings,
@@ -221,6 +421,33 @@ export function renderRunDiff(diff) {
     list('REGRESSED', diff.regressed);
     list('fixed', diff.fixed);
     list('reclassified', diff.reclassified);
+    // BATCH-33 — judge drift under its own heading, so a judge-graded suite with no `classification:`
+    // block has something to read here at all. Nothing is printed when the filter kept nothing: a
+    // quiet section IS the report on a stable re-run, and adding a reassuring "no drift" line would
+    // put a row on every run, which is the noise the filter exists to prevent.
+    if (diff.judgeDrift.length > 0) {
+        lines.push(`  JUDGE DRIFT — ${describeDriftFilter(diff.judgeDriftFilter)} (${diff.judgeDrift.length}):`);
+        for (const entry of diff.judgeDrift) {
+            const sign = entry.delta >= 0 ? '+' : '';
+            const margin = entry.after - entry.passThreshold;
+            const where = entry.reason === 'crossed'
+                ? `CROSSED the pass threshold ${entry.passThreshold}`
+                : entry.reason === 'near-threshold'
+                    ? margin === 0
+                        ? `now AT the pass threshold ${entry.passThreshold}`
+                        : margin < 0
+                            ? `now ${-margin} BELOW the pass threshold ${entry.passThreshold}`
+                            : `now ${margin} above the pass threshold ${entry.passThreshold}`
+                    : `pass threshold ${entry.passThreshold}`;
+            lines.push(`    ${entry.id}: ${entry.before} → ${entry.after} (${sign}${entry.delta}) — ${where}`);
+        }
+    }
+    if (diff.judgeDriftMean) {
+        const mean = diff.judgeDriftMean;
+        const sign = mean.delta >= 0 ? '+' : '';
+        lines.push(`  judge mean: ${mean.before.toFixed(2)} → ${mean.after.toFixed(2)} ` +
+            `(${sign}${mean.delta.toFixed(2)}) over ${mean.cases} case(s)`);
+    }
     if (diff.metricDeltas.length > 0) {
         lines.push('  metric deltas:');
         for (const delta of diff.metricDeltas) {
@@ -234,11 +461,28 @@ export function renderRunDiff(diff) {
     if (diff.regressed.length === 0 &&
         diff.fixed.length === 0 &&
         diff.reclassified.length === 0 &&
-        diff.metricDeltas.every((delta) => delta.delta === 0)) {
+        diff.metricDeltas.every((delta) => delta.delta === 0) &&
+        // BATCH-33 — drift has to count here too, or a run that printed a JUDGE DRIFT section would
+        // print "no change." underneath it and contradict itself.
+        diff.judgeDrift.length === 0 &&
+        (diff.judgeDriftMean === undefined || diff.judgeDriftMean.delta === 0)) {
         lines.push('  no change.');
     }
     return lines;
 }
+/** How the active drift filter is named in the report heading. */
+function describeDriftFilter(filter) {
+    switch (filter.mode) {
+        case 'threshold-ward':
+            return `toward the pass threshold (tolerance ${filter.tolerance})`;
+        case 'min-points':
+            return `movements of ${filter.points}+ point(s)`;
+        case 'mean':
+            return 'suite mean';
+        case 'off':
+            return 'off';
+    }
+}
 /** Does this suite declare a sweep? Small helper so the command reads declaratively. */
 export function suiteSweep(suite) {
     return suite.sweep;

package/dist/evalCompare.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalCompare.js","sourceRoot":"","sources":["../src/evalCompare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAqC9C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,IAAI,KAAK,GAA2D,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,KAAyB,CAAC;QAC9B,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/E,KAAK;YACL,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,IAAO,EACP,SAAkC;IAElC,MAAM,GAAG,GAA4B,EAAE,GAAG,IAAI,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,qBAAqB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAgB,EAAU,EAAE,CACtD,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;IAE/F,KAAK,CAAC,IAAI,CACR,GAAG,CACD,WAAW,EACX,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QACxB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CACvD,CACF,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,CACD,YAAY,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC;QACzD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,gGAAgG;IAChG,uFAAuF;IACvF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,GAAG,CACD,IAAI,EACJ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAClE,CACF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,GAAG,CACD,OAAO,GAAG,EAAE,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACnE,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,KAAK,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;6BAC6B;AAC7B,SAAS,eAAe,CACtB,MAAwB,EACxB,UAAkB,EAClB,GAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,KAAK,GAAgC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAgCD;gBACgB;AAChB,SAAS,OAAO,CAAC,MAAyC;IACxD,OAAO,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAwB,EAAE,KAAuB;IACxE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,QAAQ,IAAI,CAAC,CAAC;QAEd,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,sBAAsB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,sBAAsB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,UAAU,IAAI,CAAC,WAAW,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9E,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,YAAY;YAAE,SAAS;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,WAAW,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW;SACrF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CACX,6CAA6C,YAAY,CAAC,MAAM,yBAAyB;YACvF,GAAG,WAAW,CAAC,MAAM,4CAA4C,QAAQ,cAAc;YACvF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;QACZ,YAAY;QACZ,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,cAA2E;IAE3E,IAAI,CAAC,cAAc;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,IAAI,GAAG,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,CAAC;IAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,UAAU,CAAC,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAuB,EAAQ,EAAE;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAExC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC9C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1D,MAAM,MAAM,GACV,KAAK,CAAC,KAAK,KAAK,IAAI;gBAClB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,IACE,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAC9B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,EACrD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC"}
+{"version":3,"file":"evalCompare.js","sourceRoot":"","sources":["../src/evalCompare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AA4C9C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,IAAI,KAAK,GAA2D,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,KAAyB,CAAC;QAC9B,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/E,KAAK;YACL,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,IAAO,EACP,SAAkC;IAElC,MAAM,GAAG,GAA4B,EAAE,GAAG,IAAI,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,qBAAqB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAgB,EAAU,EAAE,CACtD,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;IAE/F,KAAK,CAAC,IAAI,CACR,GAAG,CACD,WAAW,EACX,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QACxB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CACvD,CACF,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,CACD,YAAY,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC;QACzD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,gGAAgG;IAChG,uFAAuF;IACvF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,GAAG,CACD,IAAI,EACJ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAClE,CACF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,GAAG,CACD,OAAO,GAAG,EAAE,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACnE,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,KAAK,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;6BAC6B;AAC7B,SAAS,eAAe,CACtB,MAAwB,EACxB,UAAkB,EAClB,GAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,KAAK,GAAgC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AA+CD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC;AAE/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,sEAAsE;AACtE,MAAM,CAAC,MAAM,0BAA0B,GAAqB;IAC1D,IAAI,EAAE,gBAAgB;IACtB,SAAS,EAAE,6BAA6B;CACzC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAEvB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACxC,2FAA2F;QAC3F,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,2CAA2C;gBACrF,qFAAqF;gBACrF,UAAU,CACb,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACtD,IAAI,KAAK,KAAK,SAAS;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,6BAA6B,EAAE,CAAC;QAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,yBAAyB,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,8CAA8C,yBAAyB,SAAS,IAAI,OAAO;gBACzF,sFAAsF;gBACtF,0DAA0D,CAC7D,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,CACL,gCAAgC,IAAI,mDAAmD;QACvF,sBAAsB,yBAAyB,sCAAsC,CACtF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAuB,EAAE,IAAY;IACxD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,OAAO,KAAK,CAAC;AACf,CAAC;AA8DD;;;;;;;;GAQG;AACH,SAAS,WAAW,CAAC,MAAsB;IACzC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;SACnC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;AACrE,CAAC;AAED;;sBAEsB;AACtB,SAAS,aAAa,CAAC,KAA+B;IACpD,IAAI,KAAK,EAAE,EAAE,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC;IACjC,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,WAAW,CAClB,MAAc,EACd,KAAa,EACb,aAAqB,EACrB,MAAwB;IAExB,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,gBAAgB;QAAE,OAAO,SAAS,CAAC;IACvD,IAAI,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa;QAAE,OAAO,SAAS,CAAC;IACzE,IAAI,KAAK,GAAG,MAAM,IAAI,KAAK,IAAI,aAAa,GAAG,MAAM,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC;IACzF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;gBACgB;AAChB,SAAS,OAAO,CAAC,MAAyC;IACxD,OAAO,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,QAAQ,CACtB,MAAwB,EACxB,KAAuB,EACvB,MAAM,GAAqB,0BAA0B;IAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,SAAS,GAA4E,EAAE,CAAC;IAC9F,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,QAAQ,IAAI,CAAC,CAAC;QAEd,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACxD,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG;gBACH,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,aAAa,EAAE,SAAS,CAAC,aAAa;aACvC,CAAC,CAAC;YACH,IAAI,UAAU,CAAC,aAAa,KAAK,SAAS,CAAC,aAAa;gBAAE,UAAU,IAAI,CAAC,CAAC;QAC5E,CAAC;aAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YACpC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,sBAAsB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,sBAAsB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,UAAU,IAAI,CAAC,WAAW,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9E,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,YAAY;YAAE,SAAS;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,WAAW,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW;SACrF,CAAC,CAAC;IACL,CAAC;IAED,+FAA+F;IAC/F,wFAAwF;IACxF,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,IAAI,cAA0C,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,IAAkD,EAAU,EAAE,CAC1E,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;YAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,cAAc,GAAG;gBACf,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,GAAG,UAAU;gBAC7B,KAAK,EAAE,SAAS,CAAC,MAAM;aACxB,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAChF,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,IAAI,CAAC,GAAG;gBACZ,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM;gBAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,QAAQ,CAAC,IAAI,CACX,GAAG,SAAS,gFAAgF;YAC1F,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CACX,GAAG,UAAU,gFAAgF;YAC3F,2FAA2F;YAC3F,iBAAiB,CACpB,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CACX,6CAA6C,YAAY,CAAC,MAAM,yBAAyB;YACvF,GAAG,WAAW,CAAC,MAAM,4CAA4C,QAAQ,cAAc;YACvF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;QACZ,UAAU;QACV,cAAc;QACd,gBAAgB,EAAE,MAAM;QACxB,YAAY;QACZ,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,cAA2E;IAE3E,IAAI,CAAC,cAAc;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,IAAI,GAAG,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,CAAC;IAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,UAAU,CAAC,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAuB,EAAQ,EAAE;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAExC,kGAAkG;IAClG,8FAA8F;IAC9F,gGAAgG;IAChG,2EAA2E;IAC3E,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CACR,mBAAmB,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAC7F,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YACjD,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,KAAK,SAAS;gBACxB,CAAC,CAAC,8BAA8B,KAAK,CAAC,aAAa,EAAE;gBACrD,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,gBAAgB;oBACjC,CAAC,CAAC,MAAM,KAAK,CAAC;wBACZ,CAAC,CAAC,6BAA6B,KAAK,CAAC,aAAa,EAAE;wBACpD,CAAC,CAAC,MAAM,GAAG,CAAC;4BACV,CAAC,CAAC,OAAO,CAAC,MAAM,6BAA6B,KAAK,CAAC,aAAa,EAAE;4BAClE,CAAC,CAAC,OAAO,MAAM,6BAA6B,KAAK,CAAC,aAAa,EAAE;oBACrE,CAAC,CAAC,kBAAkB,KAAK,CAAC,aAAa,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CACR,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,CAAC,KAAK,OAAO,KAAK,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CACR,iBAAiB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACnE,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,KAAK,UAAU,CACjE,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC9C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1D,MAAM,MAAM,GACV,KAAK,CAAC,KAAK,KAAK,IAAI;gBAClB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,IACE,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAC9B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC;QACrD,4FAA4F;QAC5F,0DAA0D;QAC1D,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAC5B,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK,KAAK,CAAC,CAAC,EACtE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kEAAkE;AAClE,SAAS,mBAAmB,CAAC,MAAwB;IACnD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,gBAAgB;YACnB,OAAO,wCAAwC,MAAM,CAAC,SAAS,GAAG,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,gBAAgB,MAAM,CAAC,MAAM,YAAY,CAAC;QACnD,KAAK,MAAM;YACT,OAAO,YAAY,CAAC;QACtB,KAAK,KAAK;YACR,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC"}

package/dist/evalRunner.d.ts

@@ -72,6 +72,18 @@ export interface RunEvalSuiteOptions {
  * single-turn unit keeps the proven `runCell` + {@link gradeUnit} path byte-for-byte.
  */
 export declare function runEvalSuite(suite: EvalSuite, options: RunEvalSuiteOptions): Promise<EvalSuiteSummary>;
+/**
+ * BATCH-32 — every tool name any cell of the run actually invoked: the coverage NUMERATOR.
+ *
+ * Reads the per-turn traces as well as the per-cell one, because a multi-turn cell leaves `tools`
+ * unset at the top level and carries its trace per turn — a numerator that read only the flat field
+ * would report every multi-turn suite as covering nothing, which is the failure mode most likely to
+ * be mistaken for a real coverage gap.
+ *
+ * Counts a tool that was requested or executed, error result or not: the question is which tools the
+ * suite reached. What a tool RETURNED is graded by the tool-result assertions, per case.
+ */
+export declare function collectExercisedTools(results: readonly EvalCaseResult[]): string[];
 /**
  * BATCH-25 — reduce the graded results to the shape the matrices and metrics read.
  *

package/dist/evalRunner.js

@@ -6,6 +6,7 @@ import { mechanismNeedsPermissiveRating } from '#src/evalTypes.js';
 import { UNRECOGNIZED_LABEL } from '#src/classificationTypes.js';
 import { extractClassificationValue, readRaw } from '#src/classification.js';
 import { buildClassificationReport } from '#src/classificationReport.js';
+import { computeToolCoverage } from '#src/toolCoverage.js';
 /**
  * BATCH-25 — a case's tags, read defensively.
  *
@@ -136,6 +137,12 @@ export async function runEvalSuite(suite, options) {
     // BATCH-25 — per-round classification outcomes from an injected classification target, keyed by
     // the same unique `inputIndex`. Populated only on the `options.classify` path.
     const classifyOutcomes = new Map();
+    // BATCH-32 — every cell's advertised-tool inventory, collected at the ONE point both dispatch
+    // paths pass through. Each cell resolves its own tools (a fresh MCP client per cell), so the
+    // inventories can legitimately differ and the coverage denominator is their union; collecting
+    // here rather than reading it back off the graded results keeps the multi-turn path — whose pool
+    // outcome carries no inventory of its own — from silently contributing nothing.
+    const advertisedInventories = [];
     const dispatchRunCell = async (cell) => {
         const unit = unitByInputIndex.get(cell.inputIndex);
         // BATCH-25 — a CLASSIFICATION target decides the case itself; there is no agent turn to run.
@@ -194,13 +201,21 @@ export async function runEvalSuite(suite, options) {
         }
         // Single-turn: the proven `runSingleShot`-backed path, byte-for-byte (unchanged dispatch).
         if (unit.evalCase.turns.length <= 1) {
-            return runCellFor(unit.identity)(cell);
+            const outcome = await runCellFor(unit.identity)(cell);
+            if (outcome.advertisedTools)
+                advertisedInventories.push(outcome.advertisedTools);
+            return outcome;
         }
         // Multi-turn: run the whole conversation ONCE (agent/tools built once), stash the per-turn
         // outcomes for grading, and report a synthetic pool outcome (`ok` iff every turn ran) so the
         // pool's own bookkeeping is coherent — grading reads the stash, not this outcome's answer.
         const outcomes = await runConversationFor(unit.identity)(unit.evalCase.turns.map((turn) => turn.user));
         conversationOutcomes.set(cell.inputIndex, outcomes);
+        // BATCH-32: one conversation = one agent = one inventory, repeated on every turn, so take the
+        // first turn that reports one rather than pushing N copies of the same list.
+        const conversationInventory = outcomes.find((o) => o.advertisedTools)?.advertisedTools;
+        if (conversationInventory)
+            advertisedInventories.push(conversationInventory);
         return { ok: outcomes.length > 0 && outcomes.every((o) => o.ok) };
     };
     const cellResults = await runBatchMatrix(cells, {
@@ -235,6 +250,18 @@ export async function runEvalSuite(suite, options) {
         failed: results.length - passed,
         cases: results,
     };
+    // BATCH-32 — tool coverage. Attached whenever an inventory was observed, WITHOUT waiting for the
+    // suite to opt in: "nothing in a run says how much of the surface it touched" is the whole defect
+    // this closes, so a suite that declares no `tool_coverage:` block still gets the number (it just
+    // waives nothing and gates nothing). A suite whose cells reported no inventory gets no block at
+    // all — see `computeToolCoverage` on why a `0/0` would be worse than silence.
+    const toolCoverage = computeToolCoverage({
+        inventories: advertisedInventories,
+        exercised: collectExercisedTools(results),
+        spec: suite.toolCoverage,
+    });
+    if (toolCoverage)
+        summary.toolCoverage = toolCoverage;
     // BATCH-25 — the classifier report (matrices + declared metrics) is attached ONLY for a suite that
     // declares `classification:`, so a #405-era `results.json` is byte-for-byte unchanged.
     if (spec) {
@@ -245,6 +272,29 @@ export async function runEvalSuite(suite, options) {
     }
     return summary;
 }
+/**
+ * BATCH-32 — every tool name any cell of the run actually invoked: the coverage NUMERATOR.
+ *
+ * Reads the per-turn traces as well as the per-cell one, because a multi-turn cell leaves `tools`
+ * unset at the top level and carries its trace per turn — a numerator that read only the flat field
+ * would report every multi-turn suite as covering nothing, which is the failure mode most likely to
+ * be mistaken for a real coverage gap.
+ *
+ * Counts a tool that was requested or executed, error result or not: the question is which tools the
+ * suite reached. What a tool RETURNED is graded by the tool-result assertions, per case.
+ */
+export function collectExercisedTools(results) {
+    const exercised = new Set();
+    for (const result of results) {
+        for (const tool of result.tools ?? [])
+            exercised.add(tool);
+        for (const turn of result.turns ?? []) {
+            for (const tool of turn.tools ?? [])
+                exercised.add(tool);
+        }
+    }
+    return [...exercised];
+}
 /**
  * BATCH-25 — reduce the graded results to the shape the matrices and metrics read.
  *
@@ -423,6 +473,12 @@ export function classifyEvalExit(summary) {
     if ((summary.classification?.gateFailures.length ?? 0) > 0) {
         return 1;
     }
+    // BATCH-32 — same shape, same reason: a declared coverage floor or a `require:` entry the run did
+    // not satisfy. "Every case passed" and "the suite exercised enough of the surface to mean it" are
+    // different claims, and only the second notices a 42nd tool arriving uncovered.
+    if ((summary.toolCoverage?.gateFailures.length ?? 0) > 0) {
+        return 1;
+    }
     // Every cell passed.
     return 0;
 }