@gaunt-sloth/batch 2.0.0-beta.1 → 2.0.0-beta.11

package/LICENSE

@@ -1,7 +1,7 @@
-Copyright 2025-present Andrew Kondratev
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
+Copyright 2025-present Andrew Kondratev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
 THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/dist/classificationTypes.d.ts

@@ -76,11 +76,24 @@ export interface EvalClassificationSpec {
      * `expect_action` assertion is then a parse error rather than a silently ungradeable one. */
     actionFrom?: ClassificationExtractor;
 }
-/** One field a metric predicate can read. `expected.*` is what the CORPUS declares; `actual.*` is
+/**
+ * One field a metric predicate can read. `expected.*` is what the CORPUS declares; `actual.*` is
  * what the SUT produced. The prose in a corpus plan says "label" for the corpus label and "action"
  * for the actual action taken, and that ambiguity is exactly what produces a silently-wrong metric —
- * so the surface forces the distinction to be written down. */
-export type MetricField = 'expected.label' | 'expected.action' | 'actual.label' | 'actual.action';
+ * so the surface forces the distinction to be written down.
+ *
+ * **BATCH-26 — `model.label` is the third one, and it is not a synonym for `actual.label`.** The
+ * SUT's decision and the model's judgement are the same value on most cells and different on the
+ * ones where a deterministic step raised the model's answer. A metric over `actual.label` measures
+ * the GATE (which is what a user experiences); one over `model.label` measures the RATER. Scoring
+ * the second question with the first field is what bounded the agreement figure the approvals
+ * corpus exists to produce, so both are readable and the engine says which one a metric is on.
+ *
+ * There is deliberately **no `model.action`**: the deterministic layer overrides the label and
+ * produces the action, so an "action the model chose" does not exist and a predicate on it could
+ * never match. {@link ./metrics.js parseMetricPredicate} rejects it by name.
+ */
+export type MetricField = 'expected.label' | 'expected.action' | 'actual.label' | 'actual.action' | 'model.label';
 /** The literal `none` in a predicate — matches a cell where that field is absent. */
 export declare const METRIC_NONE_LITERAL = "none";
 /**
@@ -292,6 +305,10 @@ export interface ClassifiedCell {
     expectedAction?: string;
     actualLabel?: string;
     actualAction?: string;
+    /** BATCH-26 — the judgement the MODEL rendered, read by `model.label`. Equal to
+     * {@link actualLabel} except where a deterministic step raised it; absent when nobody judged. See
+     * {@link ./evalTypes.js ClassifyOutcome.modelLabel}. */
+    modelLabel?: string;
     /** `false` = the SUT did not run / the classifier failed, so this cell has no place in any matrix
      * or denominator. It is counted as `excluded` and reported, never as a wrong answer. */
     scored: boolean;
@@ -302,6 +319,11 @@ export interface EvalCaseClassification {
     expectedAction?: string;
     actualLabel?: string;
     actualAction?: string;
+    /** BATCH-26 — the judgement the MODEL rendered, before any deterministic step overrode it. Equal
+     * to {@link actualLabel} except where one did; absent when nobody judged. Written to the cell's
+     * `results.json` so a floored case is diagnosable without re-running, and read by a `model.label`
+     * metric. See {@link ./evalTypes.js ClassifyOutcome.modelLabel}. */
+    modelLabel?: string;
     /** The raw text the extractors read, kept so an `(unrecognized)` result is diagnosable without
      * re-running. Omitted when it is identical to the cell's `answer`. */
     raw?: string;

package/dist/classificationTypes.js.map

@@ -1 +1 @@
-{"version":3,"file":"classificationTypes.js","sourceRoot":"","sources":["../src/classificationTypes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;6CAI6C;AAC7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAEnD;;wDAEwD;AACxD,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC;AA8CvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC"}
+{"version":3,"file":"classificationTypes.js","sourceRoot":"","sources":["../src/classificationTypes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;6CAI6C;AAC7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAEnD;;wDAEwD;AACxD,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC;AA4DvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC"}

package/dist/evalCompare.d.ts

@@ -71,6 +71,101 @@ export interface RunDiffEntry {
     before: string;
     after: string;
 }
+/**
+ * BATCH-33 — how a run-over-run JUDGE-SCORE drift report is filtered.
+ *
+ * ## Why there is a filter at all, and why the raw form is not a member of this union
+ *
+ * A judge score is a model's 0-10 opinion, and it wobbles between identical runs. Reporting every
+ * per-case delta therefore prints a section on every run, most of it noise, and trains the reader to
+ * skip it — at which point the real slide is skipped along with it. That is strictly WORSE than
+ * reporting no drift at all: a section nobody reads still spends the run's credibility, which is the
+ * same failure BATCH-25's untrusted metric ran into.
+ *
+ * So the raw unfiltered form is deliberately not representable. There is no `all` member, and no
+ * value of the knobs below reaches one — {@link parseJudgeDriftFilter} rejects `min:0` and bounds
+ * the threshold-ward tolerance, because a knob that admits its own degenerate value ships the form
+ * this type exists to withhold.
+ *
+ * ## What each member is for
+ *
+ * - `threshold-ward` (**the default**) — report a case only when its score CROSSED the pass
+ *   threshold, or moved DOWN to within `tolerance` points of it. Distance to the gate is the thing
+ *   worth alerting on: a 10 → 8 that stays clear of a gate at 6 is a different event from a 7 → 6
+ *   sitting on it, and only the second is about to cost a verdict. It is the default because it is
+ *   the one filter whose false-positive rate on a stable suite is near zero: at the default
+ *   tolerance a score must land at or below its own gate to report, so a corpus grading anywhere
+ *   above its gate wobbles freely and prints nothing.
+ * - `min-points` — report any movement of at least N points, either direction. The blunt
+ *   instrument: it sees large swings wherever they land, and pays for that by firing on a wobbly
+ *   judge no matter how much headroom the case had.
+ * - `mean` — report only the suite-level mean. Averaging over the corpus cancels the wobble, so it
+ *   is the quietest form of all and the one that cannot say WHICH case moved.
+ * - `off` — no drift report. Opting out is not the raw form; it prints less, not more.
+ */
+export type JudgeDriftFilter = {
+    mode: 'threshold-ward';
+    tolerance: number;
+} | {
+    mode: 'min-points';
+    points: number;
+} | {
+    mode: 'mean';
+} | {
+    mode: 'off';
+};
+/**
+ * How many points above the gate still counts as sitting ON it, when the caller names no tolerance.
+ *
+ * Zero, and the reason is a measurement rather than a preference. A local judge asked to re-rate one
+ * fixed answer against one fixed rubric twelve times returned rates spanning a full point (7 and 8,
+ * on the same input every time). So one point of movement carries no information: it is the width of
+ * the noise. A shoulder of 1 would report every cell that wobbled from 8 to 7 against the default
+ * gate of 6 — an ordinary judged suite grades right there, so the section would fire on a stable
+ * re-run, which is the one failure this node exists to avoid.
+ *
+ * At zero the rule still reports both movements the node names: a 7 → 6 lands ON the gate and
+ * reports, a 10 → 8 stays clear and does not. The shoulder was buying nothing those two clauses did
+ * not already cover, and was costing exactly the measured noise band. Widen it deliberately with
+ * `--drift threshold-ward:<n>` on a corpus whose judge is steadier than this.
+ */
+export declare const DEFAULT_JUDGE_DRIFT_TOLERANCE = 0;
+/**
+ * The widest tolerance a caller may ask for. The judge scale is 0-10 and a typical gate is 6, so a
+ * shoulder much wider than this covers the whole usable band and every downward move reports —
+ * the unfiltered form by another route, which is what the bound exists to refuse.
+ */
+export declare const MAX_JUDGE_DRIFT_TOLERANCE = 3;
+/** The filter {@link diffRuns} applies when the caller names none. */
+export declare const DEFAULT_JUDGE_DRIFT_FILTER: JudgeDriftFilter;
+/**
+ * Parse a `--drift` spec into a {@link JudgeDriftFilter}, throwing on anything unrecognised —
+ * including the degenerate values that would reconstitute the raw per-case report.
+ *
+ * Accepted: `threshold-ward` (or `threshold-ward:<tolerance>`), `min:<points>`, `mean`, `off`.
+ */
+export declare function parseJudgeDriftFilter(spec: string): JudgeDriftFilter;
+/** One case whose JUDGE SCORE moved between two runs, as kept by a {@link JudgeDriftFilter}. */
+export interface JudgeDriftEntry {
+    id: string;
+    before: number;
+    after: number;
+    /** `after - before`. Negative is a slide toward the gate. */
+    delta: number;
+    /** The gate the movement is measured against — THIS run's, since that is the one now in force. */
+    passThreshold: number;
+    /** Why the filter kept it: the score crossed the gate, slid onto the gate's shoulder, or simply
+     * moved far enough for a `min-points` filter. */
+    reason: 'crossed' | 'near-threshold' | 'moved';
+}
+/** The suite-level judge-score mean — the `mean` filter's whole output. */
+export interface JudgeDriftMean {
+    before: number;
+    after: number;
+    delta: number;
+    /** Cases carrying a rate on BOTH sides, i.e. the mean's denominator. */
+    cases: number;
+}
 /** The run-over-run diff. */
 export interface RunDiff {
     /** Cases in both runs. */
@@ -88,6 +183,20 @@ export interface RunDiff {
         after: number | null;
         delta: number | null;
     }[];
+    /**
+     * BATCH-33 — per-case judge-score movements the active filter kept. The signal a VERDICT-only
+     * diff structurally cannot see: a judge-graded suite with no `classification:` block has neither
+     * `reclassified` nor `metricDeltas`, so before this its diff was strictly binary and a score
+     * sliding toward its gate reported "no change." until the run it finally broke.
+     *
+     * EMPTY under the `mean` and `off` filters, which report no per-case rows at all.
+     */
+    judgeDrift: JudgeDriftEntry[];
+    /** The suite-level mean, under the `mean` filter only. */
+    judgeDriftMean?: JudgeDriftMean;
+    /** The filter that produced the two fields above, echoed so the render can name it: a quiet drift
+     * section means nothing unless the reader can see WHICH filter was quiet. */
+    judgeDriftFilter: JudgeDriftFilter;
     /** Ids in only one of the two runs — reported, so a shrunken corpus cannot read as "no change". */
     onlyInBefore: string[];
     onlyInAfter: string[];
@@ -100,8 +209,15 @@ export interface RunDiff {
  * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
  * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
  * and that is exactly the drift a pass-rate comparison cannot see.
+ *
+ * A fourth, {@link RunDiff.judgeDrift}, covers the suites the other three cannot: `reclassified`
+ * and `metricDeltas` both need a `classification:` block, so an ordinary judge-graded corpus got a
+ * strictly binary diff and learned nothing until a score finally broke its gate. `filter` decides
+ * how much of that movement is worth printing — see {@link JudgeDriftFilter} for why it defaults to
+ * the threshold-ward one and why the unfiltered form is not on offer. **The default lives here, not
+ * only in the CLI**, so every caller gets the quiet behaviour without having to know to ask.
  */
-export declare function diffRuns(before: EvalSuiteSummary, after: EvalSuiteSummary): RunDiff;
+export declare function diffRuns(before: EvalSuiteSummary, after: EvalSuiteSummary, filter?: JudgeDriftFilter): RunDiff;
 /** Render a {@link RunDiff} as plain lines. */
 export declare function renderRunDiff(diff: RunDiff): string[];
 /** Does this suite declare a sweep? Small helper so the command reads declaratively. */

package/dist/evalCompare.js

@@ -128,6 +128,137 @@ function cellMetricValue(column, metricName, tag) {
         return 'n/a';
     return formatTally(tally);
 }
+/**
+ * How many points above the gate still counts as sitting ON it, when the caller names no tolerance.
+ *
+ * Zero, and the reason is a measurement rather than a preference. A local judge asked to re-rate one
+ * fixed answer against one fixed rubric twelve times returned rates spanning a full point (7 and 8,
+ * on the same input every time). So one point of movement carries no information: it is the width of
+ * the noise. A shoulder of 1 would report every cell that wobbled from 8 to 7 against the default
+ * gate of 6 — an ordinary judged suite grades right there, so the section would fire on a stable
+ * re-run, which is the one failure this node exists to avoid.
+ *
+ * At zero the rule still reports both movements the node names: a 7 → 6 lands ON the gate and
+ * reports, a 10 → 8 stays clear and does not. The shoulder was buying nothing those two clauses did
+ * not already cover, and was costing exactly the measured noise band. Widen it deliberately with
+ * `--drift threshold-ward:<n>` on a corpus whose judge is steadier than this.
+ */
+export const DEFAULT_JUDGE_DRIFT_TOLERANCE = 0;
+/**
+ * The widest tolerance a caller may ask for. The judge scale is 0-10 and a typical gate is 6, so a
+ * shoulder much wider than this covers the whole usable band and every downward move reports —
+ * the unfiltered form by another route, which is what the bound exists to refuse.
+ */
+export const MAX_JUDGE_DRIFT_TOLERANCE = 3;
+/** The filter {@link diffRuns} applies when the caller names none. */
+export const DEFAULT_JUDGE_DRIFT_FILTER = {
+    mode: 'threshold-ward',
+    tolerance: DEFAULT_JUDGE_DRIFT_TOLERANCE,
+};
+/**
+ * Parse a `--drift` spec into a {@link JudgeDriftFilter}, throwing on anything unrecognised —
+ * including the degenerate values that would reconstitute the raw per-case report.
+ *
+ * Accepted: `threshold-ward` (or `threshold-ward:<tolerance>`), `min:<points>`, `mean`, `off`.
+ */
+export function parseJudgeDriftFilter(spec) {
+    const parts = spec.trim().split(':');
+    if (parts.length > 2)
+        throw new Error(driftSpecError(spec));
+    const mode = parts[0].trim().toLowerCase();
+    const value = parts[1];
+    if (mode === 'off' || mode === 'mean') {
+        if (value !== undefined)
+            throw new Error(driftSpecError(spec));
+        return mode === 'off' ? { mode: 'off' } : { mode: 'mean' };
+    }
+    if (mode === 'min') {
+        const points = driftNumber(value, spec);
+        // 0 or less would keep every movement — the unfiltered report this filter exists to avoid.
+        if (points < 1 || points > 10) {
+            throw new Error(`--drift min:<points> takes 1-10, so "${spec}" is not accepted: it would report every ` +
+                'judge-score movement, which is the unfiltered form that trains readers to skip the ' +
+                'section.');
+        }
+        return { mode: 'min-points', points };
+    }
+    if (mode === 'threshold-ward' || mode === 'threshold') {
+        if (value === undefined)
+            return { mode: 'threshold-ward', tolerance: DEFAULT_JUDGE_DRIFT_TOLERANCE };
+        const tolerance = driftNumber(value, spec);
+        if (tolerance < 0 || tolerance > MAX_JUDGE_DRIFT_TOLERANCE) {
+            throw new Error(`--drift threshold-ward:<tolerance> takes 0-${MAX_JUDGE_DRIFT_TOLERANCE}, so "${spec}" is ` +
+                'not accepted: a shoulder that wide reports nearly every downward move, which is the ' +
+                'unfiltered form that trains readers to skip the section.');
+        }
+        return { mode: 'threshold-ward', tolerance };
+    }
+    throw new Error(driftSpecError(spec));
+}
+function driftSpecError(spec) {
+    return (`unrecognised --drift filter "${spec}". Use "threshold-ward" (the default, optionally ` +
+        `"threshold-ward:<0-${MAX_JUDGE_DRIFT_TOLERANCE}>"), "min:<1-10>", "mean", or "off".`);
+}
+function driftNumber(raw, spec) {
+    if (raw === undefined || raw.trim() === '')
+        throw new Error(driftSpecError(spec));
+    const value = Number(raw.trim());
+    if (!Number.isInteger(value))
+        throw new Error(driftSpecError(spec));
+    return value;
+}
+/**
+ * The judge rate representing one CELL, or `undefined` when no verdict was rendered for it.
+ *
+ * A single-turn cell carries its own {@link EvalCaseResult.judge}. A MULTI-TURN cell leaves that
+ * unset and grades per turn, so its representative rate is the LOWEST any turn scored: the cell
+ * passes iff every turn passes, which makes the weakest turn the one sitting nearest the gate and
+ * therefore the one this report is about. Reading only the top-level field would hand every
+ * multi-turn suite an empty drift section forever — the same silence this node exists to end.
+ */
+function judgeRateOf(result) {
+    const direct = rateOfOutcome(result.judge);
+    if (direct !== undefined)
+        return direct;
+    const turnRates = (result.turns ?? [])
+        .map((turn) => rateOfOutcome(turn.judge))
+        .filter((rate) => rate !== undefined);
+    return turnRates.length === 0 ? undefined : Math.min(...turnRates);
+}
+/** A usable 0-10 rate, or `undefined`. A judge that errored, timed out, returned something
+ * unparseable or was never asked has no score to compare, and substituting one would report a slide
+ * nobody measured. */
+function rateOfOutcome(judge) {
+    if (judge?.ok !== true)
+        return undefined;
+    const rate = judge.verdict?.rate;
+    return typeof rate === 'number' && Number.isFinite(rate) ? rate : undefined;
+}
+/**
+ * Does one movement survive the filter? Returns the reason it was kept, or `undefined`.
+ *
+ * The threshold-ward rule is two clauses and BOTH are load-bearing:
+ *
+ * - a CROSSING of the gate, either direction, always reports. That is the moment the score changed
+ *   what it is worth, and its numbers are what explain the verdict flip printed beside it.
+ * - otherwise, only a DOWNWARD move landing within `tolerance` of the gate. Direction alone is not
+ *   the rule — a 10 → 8 is downward and still clear of a gate at 6, and keeping it is exactly how
+ *   the section fills with wobble. Proximity alone is not the rule either, or a case recovering
+ *   6 → 7 would report. It is the pair, and a reader tempted to simplify one clause away should
+ *   expect the section to go noisy on the next stable re-run.
+ */
+function driftReason(before, after, passThreshold, filter) {
+    if (filter.mode === 'min-points') {
+        return Math.abs(after - before) >= filter.points ? 'moved' : undefined;
+    }
+    if (filter.mode !== 'threshold-ward')
+        return undefined;
+    if (before >= passThreshold !== after >= passThreshold)
+        return 'crossed';
+    if (after < before && after <= passThreshold + filter.tolerance)
+        return 'near-threshold';
+    return undefined;
+}
 /** The key a cell is diffed on across runs: id plus identity, since a matrix cell's id alone is
  * ambiguous. */
 function diffKey(result) {
@@ -140,19 +271,44 @@ function diffKey(result) {
  * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
  * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
  * and that is exactly the drift a pass-rate comparison cannot see.
+ *
+ * A fourth, {@link RunDiff.judgeDrift}, covers the suites the other three cannot: `reclassified`
+ * and `metricDeltas` both need a `classification:` block, so an ordinary judge-graded corpus got a
+ * strictly binary diff and learned nothing until a score finally broke its gate. `filter` decides
+ * how much of that movement is worth printing — see {@link JudgeDriftFilter} for why it defaults to
+ * the threshold-ward one and why the unfiltered form is not on offer. **The default lives here, not
+ * only in the CLI**, so every caller gets the quiet behaviour without having to know to ask.
  */
-export function diffRuns(before, after) {
+export function diffRuns(before, after, filter = DEFAULT_JUDGE_DRIFT_FILTER) {
     const beforeByKey = new Map(before.cases.map((result) => [diffKey(result), result]));
     const afterByKey = new Map(after.cases.map((result) => [diffKey(result), result]));
     const regressed = [];
     const fixed = [];
     const reclassified = [];
+    const ratePairs = [];
+    let lostRates = 0;
+    let movedGates = 0;
     let compared = 0;
     for (const [key, afterCase] of afterByKey) {
         const beforeCase = beforeByKey.get(key);
         if (!beforeCase)
             continue;
         compared += 1;
+        const beforeRate = judgeRateOf(beforeCase);
+        const afterRate = judgeRateOf(afterCase);
+        if (beforeRate !== undefined && afterRate !== undefined) {
+            ratePairs.push({
+                key,
+                before: beforeRate,
+                after: afterRate,
+                passThreshold: afterCase.passThreshold,
+            });
+            if (beforeCase.passThreshold !== afterCase.passThreshold)
+                movedGates += 1;
+        }
+        else if (beforeRate !== undefined) {
+            lostRates += 1;
+        }
         if (beforeCase.verdict === 'PASS' && afterCase.verdict === 'FAIL') {
             regressed.push({ id: key, before: 'PASS', after: 'FAIL' });
         }
@@ -181,7 +337,48 @@ export function diffRuns(before, after) {
             delta: beforeValue === null || afterValue === null ? null : afterValue - beforeValue,
         });
     }
+    // BATCH-33 — the judge-score drift, as much of it as the filter keeps. `mean` reports only the
+    // suite aggregate (no per-case rows); every other filter reports rows and no aggregate.
+    const judgeDrift = [];
+    let judgeDriftMean;
+    if (filter.mode === 'mean') {
+        if (ratePairs.length > 0) {
+            const mean = (pick) => ratePairs.reduce((sum, pair) => sum + pick(pair), 0) / ratePairs.length;
+            const meanBefore = mean((pair) => pair.before);
+            const meanAfter = mean((pair) => pair.after);
+            judgeDriftMean = {
+                before: meanBefore,
+                after: meanAfter,
+                delta: meanAfter - meanBefore,
+                cases: ratePairs.length,
+            };
+        }
+    }
+    else {
+        for (const pair of ratePairs) {
+            const reason = driftReason(pair.before, pair.after, pair.passThreshold, filter);
+            if (reason === undefined)
+                continue;
+            judgeDrift.push({
+                id: pair.key,
+                before: pair.before,
+                after: pair.after,
+                delta: pair.after - pair.before,
+                passThreshold: pair.passThreshold,
+                reason,
+            });
+        }
+    }
     const warnings = [];
+    if (lostRates > 0 && filter.mode !== 'off') {
+        warnings.push(`${lostRates} case(s) were judged in the baseline and produced no judge score in this run, ` +
+            'so their drift could not be computed. A quiet drift section here is not "the scores held".');
+    }
+    if (movedGates > 0 && filter.mode !== 'off') {
+        warnings.push(`${movedGates} case(s) changed their pass threshold between the two runs. Drift is measured ` +
+            "against THIS run's gate, so some of the distance reported here is the gate moving rather " +
+            'than the score.');
+    }
     if (onlyInBefore.length > 0 || onlyInAfter.length > 0) {
         warnings.push(`the two runs do not cover the same cases: ${onlyInBefore.length} only in the baseline, ` +
             `${onlyInAfter.length} only in this run. The comparison covers ${compared} case(s); a ` +
@@ -193,6 +390,9 @@ export function diffRuns(before, after) {
         fixed,
         reclassified,
         metricDeltas,
+        judgeDrift,
+        judgeDriftMean,
+        judgeDriftFilter: filter,
         onlyInBefore,
         onlyInAfter,
         warnings,
@@ -221,6 +421,33 @@ export function renderRunDiff(diff) {
     list('REGRESSED', diff.regressed);
     list('fixed', diff.fixed);
     list('reclassified', diff.reclassified);
+    // BATCH-33 — judge drift under its own heading, so a judge-graded suite with no `classification:`
+    // block has something to read here at all. Nothing is printed when the filter kept nothing: a
+    // quiet section IS the report on a stable re-run, and adding a reassuring "no drift" line would
+    // put a row on every run, which is the noise the filter exists to prevent.
+    if (diff.judgeDrift.length > 0) {
+        lines.push(`  JUDGE DRIFT — ${describeDriftFilter(diff.judgeDriftFilter)} (${diff.judgeDrift.length}):`);
+        for (const entry of diff.judgeDrift) {
+            const sign = entry.delta >= 0 ? '+' : '';
+            const margin = entry.after - entry.passThreshold;
+            const where = entry.reason === 'crossed'
+                ? `CROSSED the pass threshold ${entry.passThreshold}`
+                : entry.reason === 'near-threshold'
+                    ? margin === 0
+                        ? `now AT the pass threshold ${entry.passThreshold}`
+                        : margin < 0
+                            ? `now ${-margin} BELOW the pass threshold ${entry.passThreshold}`
+                            : `now ${margin} above the pass threshold ${entry.passThreshold}`
+                    : `pass threshold ${entry.passThreshold}`;
+            lines.push(`    ${entry.id}: ${entry.before} → ${entry.after} (${sign}${entry.delta}) — ${where}`);
+        }
+    }
+    if (diff.judgeDriftMean) {
+        const mean = diff.judgeDriftMean;
+        const sign = mean.delta >= 0 ? '+' : '';
+        lines.push(`  judge mean: ${mean.before.toFixed(2)} → ${mean.after.toFixed(2)} ` +
+            `(${sign}${mean.delta.toFixed(2)}) over ${mean.cases} case(s)`);
+    }
     if (diff.metricDeltas.length > 0) {
         lines.push('  metric deltas:');
         for (const delta of diff.metricDeltas) {
@@ -234,11 +461,28 @@ export function renderRunDiff(diff) {
     if (diff.regressed.length === 0 &&
         diff.fixed.length === 0 &&
         diff.reclassified.length === 0 &&
-        diff.metricDeltas.every((delta) => delta.delta === 0)) {
+        diff.metricDeltas.every((delta) => delta.delta === 0) &&
+        // BATCH-33 — drift has to count here too, or a run that printed a JUDGE DRIFT section would
+        // print "no change." underneath it and contradict itself.
+        diff.judgeDrift.length === 0 &&
+        (diff.judgeDriftMean === undefined || diff.judgeDriftMean.delta === 0)) {
         lines.push('  no change.');
     }
     return lines;
 }
+/** How the active drift filter is named in the report heading. */
+function describeDriftFilter(filter) {
+    switch (filter.mode) {
+        case 'threshold-ward':
+            return `toward the pass threshold (tolerance ${filter.tolerance})`;
+        case 'min-points':
+            return `movements of ${filter.points}+ point(s)`;
+        case 'mean':
+            return 'suite mean';
+        case 'off':
+            return 'off';
+    }
+}
 /** Does this suite declare a sweep? Small helper so the command reads declaratively. */
 export function suiteSweep(suite) {
     return suite.sweep;

package/dist/evalCompare.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalCompare.js","sourceRoot":"","sources":["../src/evalCompare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAqC9C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,IAAI,KAAK,GAA2D,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,KAAyB,CAAC;QAC9B,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/E,KAAK;YACL,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,IAAO,EACP,SAAkC;IAElC,MAAM,GAAG,GAA4B,EAAE,GAAG,IAAI,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,qBAAqB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAgB,EAAU,EAAE,CACtD,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;IAE/F,KAAK,CAAC,IAAI,CACR,GAAG,CACD,WAAW,EACX,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QACxB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CACvD,CACF,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,CACD,YAAY,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC;QACzD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,gGAAgG;IAChG,uFAAuF;IACvF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,GAAG,CACD,IAAI,EACJ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAClE,CACF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,GAAG,CACD,OAAO,GAAG,EAAE,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACnE,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,KAAK,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;6BAC6B;AAC7B,SAAS,eAAe,CACtB,MAAwB,EACxB,UAAkB,EAClB,GAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,KAAK,GAAgC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAgCD;gBACgB;AAChB,SAAS,OAAO,CAAC,MAAyC;IACxD,OAAO,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAwB,EAAE,KAAuB;IACxE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,QAAQ,IAAI,CAAC,CAAC;QAEd,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,sBAAsB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,sBAAsB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,UAAU,IAAI,CAAC,WAAW,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9E,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,YAAY;YAAE,SAAS;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,WAAW,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW;SACrF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CACX,6CAA6C,YAAY,CAAC,MAAM,yBAAyB;YACvF,GAAG,WAAW,CAAC,MAAM,4CAA4C,QAAQ,cAAc;YACvF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;QACZ,YAAY;QACZ,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,cAA2E;IAE3E,IAAI,CAAC,cAAc;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,IAAI,GAAG,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,CAAC;IAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,UAAU,CAAC,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAuB,EAAQ,EAAE;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAExC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC9C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1D,MAAM,MAAM,GACV,KAAK,CAAC,KAAK,KAAK,IAAI;gBAClB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,IACE,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAC9B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,EACrD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC"}
+{"version":3,"file":"evalCompare.js","sourceRoot":"","sources":["../src/evalCompare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AA4C9C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,IAAI,KAAK,GAA2D,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,KAAyB,CAAC;QAC9B,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/E,KAAK;YACL,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,IAAO,EACP,SAAkC;IAElC,MAAM,GAAG,GAA4B,EAAE,GAAG,IAAI,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,qBAAqB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAgB,EAAU,EAAE,CACtD,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;IAE/F,KAAK,CAAC,IAAI,CACR,GAAG,CACD,WAAW,EACX,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QACxB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CACvD,CACF,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,CACD,YAAY,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC;QACzD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,gGAAgG;IAChG,uFAAuF;IACvF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,GAAG,CACD,IAAI,EACJ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAClE,CACF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,GAAG,CACD,OAAO,GAAG,EAAE,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACnE,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,KAAK,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;6BAC6B;AAC7B,SAAS,eAAe,CACtB,MAAwB,EACxB,UAAkB,EAClB,GAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,KAAK,GAAgC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AA+CD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC;AAE/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,sEAAsE;AACtE,MAAM,CAAC,MAAM,0BAA0B,GAAqB;IAC1D,IAAI,EAAE,gBAAgB;IACtB,SAAS,EAAE,6BAA6B;CACzC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAEvB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACxC,2FAA2F;QAC3F,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,wCAAwC,IAAI,2CAA2C;gBACrF,qFAAqF;gBACrF,UAAU,CACb,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,IAAI,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACtD,IAAI,KAAK,KAAK,SAAS;YACrB,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,6BAA6B,EAAE,CAAC;QAC9E,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,yBAAyB,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,8CAA8C,yBAAyB,SAAS,IAAI,OAAO;gBACzF,sFAAsF;gBACtF,0DAA0D,CAC7D,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,CACL,gCAAgC,IAAI,mDAAmD;QACvF,sBAAsB,yBAAyB,sCAAsC,CACtF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,GAAuB,EAAE,IAAY;IACxD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,OAAO,KAAK,CAAC;AACf,CAAC;AA8DD;;;;;;;;GAQG;AACH,SAAS,WAAW,CAAC,MAAsB;IACzC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;SACnC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;AACrE,CAAC;AAED;;sBAEsB;AACtB,SAAS,aAAa,CAAC,KAA+B;IACpD,IAAI,KAAK,EAAE,EAAE,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC;IACjC,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,WAAW,CAClB,MAAc,EACd,KAAa,EACb,aAAqB,EACrB,MAAwB;IAExB,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,gBAAgB;QAAE,OAAO,SAAS,CAAC;IACvD,IAAI,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa;QAAE,OAAO,SAAS,CAAC;IACzE,IAAI,KAAK,GAAG,MAAM,IAAI,KAAK,IAAI,aAAa,GAAG,MAAM,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC;IACzF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;gBACgB;AAChB,SAAS,OAAO,CAAC,MAAyC;IACxD,OAAO,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,QAAQ,CACtB,MAAwB,EACxB,KAAuB,EACvB,MAAM,GAAqB,0BAA0B;IAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,MAAM,SAAS,GAA4E,EAAE,CAAC;IAC9F,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,QAAQ,IAAI,CAAC,CAAC;QAEd,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACxD,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG;gBACH,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,aAAa,EAAE,SAAS,CAAC,aAAa;aACvC,CAAC,CAAC;YACH,IAAI,UAAU,CAAC,aAAa,KAAK,SAAS,CAAC,aAAa;gBAAE,UAAU,IAAI,CAAC,CAAC;QAC5E,CAAC;aAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YACpC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,sBAAsB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,sBAAsB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,UAAU,IAAI,CAAC,WAAW,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9E,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,YAAY;YAAE,SAAS;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,WAAW,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW;SACrF,CAAC,CAAC;IACL,CAAC;IAED,+FAA+F;IAC/F,wFAAwF;IACxF,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,IAAI,cAA0C,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,IAAkD,EAAU,EAAE,CAC1E,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;YAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,cAAc,GAAG;gBACf,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,SAAS,GAAG,UAAU;gBAC7B,KAAK,EAAE,SAAS,CAAC,MAAM;aACxB,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAChF,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,UAAU,CAAC,IAAI,CAAC;gBACd,EAAE,EAAE,IAAI,CAAC,GAAG;gBACZ,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM;gBAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,QAAQ,CAAC,IAAI,CACX,GAAG,SAAS,gFAAgF;YAC1F,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CACX,GAAG,UAAU,gFAAgF;YAC3F,2FAA2F;YAC3F,iBAAiB,CACpB,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CACX,6CAA6C,YAAY,CAAC,MAAM,yBAAyB;YACvF,GAAG,WAAW,CAAC,MAAM,4CAA4C,QAAQ,cAAc;YACvF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;QACZ,UAAU;QACV,cAAc;QACd,gBAAgB,EAAE,MAAM;QACxB,YAAY;QACZ,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,cAA2E;IAE3E,IAAI,CAAC,cAAc;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,IAAI,GAAG,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,CAAC;IAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,UAAU,CAAC,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAuB,EAAQ,EAAE;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAExC,kGAAkG;IAClG,8FAA8F;IAC9F,gGAAgG;IAChG,2EAA2E;IAC3E,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CACR,mBAAmB,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAC7F,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC;YACjD,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,KAAK,SAAS;gBACxB,CAAC,CAAC,8BAA8B,KAAK,CAAC,aAAa,EAAE;gBACrD,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,gBAAgB;oBACjC,CAAC,CAAC,MAAM,KAAK,CAAC;wBACZ,CAAC,CAAC,6BAA6B,KAAK,CAAC,aAAa,EAAE;wBACpD,CAAC,CAAC,MAAM,GAAG,CAAC;4BACV,CAAC,CAAC,OAAO,CAAC,MAAM,6BAA6B,KAAK,CAAC,aAAa,EAAE;4BAClE,CAAC,CAAC,OAAO,MAAM,6BAA6B,KAAK,CAAC,aAAa,EAAE;oBACrE,CAAC,CAAC,kBAAkB,KAAK,CAAC,aAAa,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CACR,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,CAAC,KAAK,OAAO,KAAK,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CACR,iBAAiB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACnE,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,KAAK,UAAU,CACjE,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC9C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1D,MAAM,MAAM,GACV,KAAK,CAAC,KAAK,KAAK,IAAI;gBAClB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,IACE,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAC9B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC;QACrD,4FAA4F;QAC5F,0DAA0D;QAC1D,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QAC5B,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK,KAAK,CAAC,CAAC,EACtE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kEAAkE;AAClE,SAAS,mBAAmB,CAAC,MAAwB;IACnD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,gBAAgB;YACnB,OAAO,wCAAwC,MAAM,CAAC,SAAS,GAAG,CAAC;QACrE,KAAK,YAAY;YACf,OAAO,gBAAgB,MAAM,CAAC,MAAM,YAAY,CAAC;QACnD,KAAK,MAAM;YACT,OAAO,YAAY,CAAC;QACtB,KAAK,KAAK;YACR,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC"}

package/dist/evalRunner.d.ts

@@ -72,6 +72,18 @@ export interface RunEvalSuiteOptions {
  * single-turn unit keeps the proven `runCell` + {@link gradeUnit} path byte-for-byte.
  */
 export declare function runEvalSuite(suite: EvalSuite, options: RunEvalSuiteOptions): Promise<EvalSuiteSummary>;
+/**
+ * BATCH-32 — every tool name any cell of the run actually invoked: the coverage NUMERATOR.
+ *
+ * Reads the per-turn traces as well as the per-cell one, because a multi-turn cell leaves `tools`
+ * unset at the top level and carries its trace per turn — a numerator that read only the flat field
+ * would report every multi-turn suite as covering nothing, which is the failure mode most likely to
+ * be mistaken for a real coverage gap.
+ *
+ * Counts a tool that was requested or executed, error result or not: the question is which tools the
+ * suite reached. What a tool RETURNED is graded by the tool-result assertions, per case.
+ */
+export declare function collectExercisedTools(results: readonly EvalCaseResult[]): string[];
 /**
  * BATCH-25 — reduce the graded results to the shape the matrices and metrics read.
  *