@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.26

package/dist/evalCompare.js

@@ -0,0 +1,246 @@
+import { formatTally } from '#src/metrics.js';
+/**
+ * Expand a {@link EvalSweep} into its cartesian product of cells, in declared axis order.
+ *
+ * A later axis wins on a conflicting key, and that is documented rather than defended against: two
+ * axes that set the same config key are describing the same knob twice, which the author should see
+ * in the cell name.
+ */
+export function expandSweep(sweep) {
+    let cells = [{ parts: [] }];
+    for (const axis of sweep.axes) {
+        const next = [];
+        for (const cell of cells) {
+            for (const value of axis.values) {
+                next.push({ parts: [...cell.parts, { axis: axis.name, value }] });
+            }
+        }
+        cells = next;
+    }
+    return cells.map((cell) => {
+        let model;
+        let config = {};
+        for (const part of cell.parts) {
+            if (part.value.model !== undefined)
+                model = part.value.model;
+            if (part.value.config)
+                config = deepMerge(config, part.value.config);
+        }
+        return {
+            name: cell.parts.map((part) => `${part.axis}=${part.value.name}`).join(' · '),
+            dirName: cell.parts.map((part) => `${part.axis}-${part.value.name}`).join('__'),
+            model,
+            config,
+        };
+    });
+}
+/**
+ * Deep-merge plain data (the sweep's `config:` overrides) into a target.
+ *
+ * Objects merge recursively; arrays and scalars REPLACE. Replacing an array is the right default
+ * for config: an override that appended to `mcpServers` or `allowedTools` would silently keep the
+ * base value the author meant to displace, which is the harder bug to see.
+ *
+ * Prototype-polluting keys are skipped — a suite file is only semi-trusted input.
+ */
+export function deepMerge(base, overrides) {
+    const out = { ...base };
+    for (const [key, value] of Object.entries(overrides)) {
+        if (key === '__proto__' || key === 'constructor' || key === 'prototype')
+            continue;
+        const existing = out[key];
+        if (isPlainObject(existing) && isPlainObject(value)) {
+            out[key] = deepMerge(existing, value);
+        }
+        else {
+            out[key] = value;
+        }
+    }
+    return out;
+}
+function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Render the cross-cell comparison table.
+ *
+ * Rows are the metrics the suite declares (plus pass rate), columns are the sweep cells. A metric
+ * that a cell could not compute renders `n/a`, never a blank and never a zero — the same rule the
+ * metric engine follows, for the same reason.
+ *
+ * Per-tag sub-scores get their own rows under each metric, because the whole point of running a
+ * sweep is to see which setting moved which family, and a blended per-cell number cannot show that.
+ */
+export function renderComparison(columns) {
+    if (columns.length === 0)
+        return [];
+    const lines = ['', `COMPARISON across ${columns.length} cell(s)`];
+    const nameWidth = Math.max(...columns.map((column) => column.name.length), 'metric'.length, 24);
+    const columnWidth = Math.max(...columns.map((column) => column.name.length), 16) + 2;
+    const header = columns.map((column) => column.name.padStart(columnWidth)).join('');
+    lines.push(`  ${'metric'.padEnd(nameWidth)}${header}`);
+    const row = (label, values) => `  ${label.padEnd(nameWidth)}${values.map((value) => value.padStart(columnWidth)).join('')}`;
+    lines.push(row('pass rate', columns.map((column) => column.summary.total === 0
+        ? 'n/a (0 cases)'
+        : `${column.summary.passed}/${column.summary.total}`)));
+    lines.push(row('classified', columns.map((column) => {
+        const coverage = column.summary.classification?.coverage;
+        return coverage ? `${coverage.scored}/${coverage.total}` : 'n/a';
+    })));
+    // Metric rows, in the order the FIRST cell declares them, then any metric only later cells have
+    // (which would mean the cells ran different suites — worth seeing rather than hiding).
+    const metricNames = [];
+    for (const column of columns) {
+        for (const metric of column.summary.classification?.metrics ?? []) {
+            if (!metricNames.includes(metric.name))
+                metricNames.push(metric.name);
+        }
+    }
+    const tags = [];
+    for (const column of columns) {
+        for (const tag of column.summary.classification?.tags ?? []) {
+            if (!tags.includes(tag))
+                tags.push(tag);
+        }
+    }
+    for (const name of metricNames) {
+        lines.push('');
+        lines.push(row(name, columns.map((column) => cellMetricValue(column, name, undefined))));
+        for (const tag of tags) {
+            lines.push(row(`  · ${tag}`, columns.map((column) => cellMetricValue(column, name, tag))));
+        }
+        // A gate breach must be visible in the comparison itself, not only in the per-cell report.
+        const breached = columns.filter((column) => (column.summary.classification?.gateFailures ?? []).includes(name));
+        if (breached.length > 0) {
+            lines.push(`  ${''.padEnd(nameWidth)}GATE FAILED in: ${breached.map((c) => c.name).join(', ')}`);
+        }
+    }
+    return lines;
+}
+/** One cell's value for a metric (overall, or for one tag). `n/a` when the cell has no such metric
+ * — never blank, never 0. */
+function cellMetricValue(column, metricName, tag) {
+    const metric = column.summary.classification?.metrics.find((m) => m.name === metricName);
+    if (!metric)
+        return 'n/a';
+    const tally = tag === undefined ? metric.overall : metric.byTag[tag];
+    if (!tally)
+        return 'n/a';
+    return formatTally(tally);
+}
+/** The key a cell is diffed on across runs: id plus identity, since a matrix cell's id alone is
+ * ambiguous. */
+function diffKey(result) {
+    return result.identity === undefined ? result.id : `${result.id}__${result.identity}`;
+}
+/**
+ * Diff two runs of the same suite, so a rating-prompt edit produces a signal rather than a vibe.
+ *
+ * Three separate lists, because they answer different questions: verdict regressions are what a CI
+ * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
+ * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
+ * and that is exactly the drift a pass-rate comparison cannot see.
+ */
+export function diffRuns(before, after) {
+    const beforeByKey = new Map(before.cases.map((result) => [diffKey(result), result]));
+    const afterByKey = new Map(after.cases.map((result) => [diffKey(result), result]));
+    const regressed = [];
+    const fixed = [];
+    const reclassified = [];
+    let compared = 0;
+    for (const [key, afterCase] of afterByKey) {
+        const beforeCase = beforeByKey.get(key);
+        if (!beforeCase)
+            continue;
+        compared += 1;
+        if (beforeCase.verdict === 'PASS' && afterCase.verdict === 'FAIL') {
+            regressed.push({ id: key, before: 'PASS', after: 'FAIL' });
+        }
+        else if (beforeCase.verdict === 'FAIL' && afterCase.verdict === 'PASS') {
+            fixed.push({ id: key, before: 'FAIL', after: 'PASS' });
+        }
+        const beforeClass = describeClassification(beforeCase.classification);
+        const afterClass = describeClassification(afterCase.classification);
+        if (beforeClass !== afterClass && (beforeClass !== '-' || afterClass !== '-')) {
+            reclassified.push({ id: key, before: beforeClass, after: afterClass });
+        }
+    }
+    const onlyInBefore = [...beforeByKey.keys()].filter((key) => !afterByKey.has(key));
+    const onlyInAfter = [...afterByKey.keys()].filter((key) => !beforeByKey.has(key));
+    const metricDeltas = [];
+    for (const afterMetric of after.classification?.metrics ?? []) {
+        const beforeMetric = before.classification?.metrics.find((m) => m.name === afterMetric.name);
+        if (!beforeMetric)
+            continue;
+        const beforeValue = beforeMetric.overall.value;
+        const afterValue = afterMetric.overall.value;
+        metricDeltas.push({
+            name: afterMetric.name,
+            before: beforeValue,
+            after: afterValue,
+            delta: beforeValue === null || afterValue === null ? null : afterValue - beforeValue,
+        });
+    }
+    const warnings = [];
+    if (onlyInBefore.length > 0 || onlyInAfter.length > 0) {
+        warnings.push(`the two runs do not cover the same cases: ${onlyInBefore.length} only in the baseline, ` +
+            `${onlyInAfter.length} only in this run. The comparison covers ${compared} case(s); a ` +
+            'case that disappeared cannot regress, so "no regressions" here is not "nothing broke".');
+    }
+    return {
+        compared,
+        regressed,
+        fixed,
+        reclassified,
+        metricDeltas,
+        onlyInBefore,
+        onlyInAfter,
+        warnings,
+    };
+}
+function describeClassification(classification) {
+    if (!classification)
+        return '-';
+    const label = classification.actualLabel ?? '-';
+    const action = classification.actualAction;
+    return action === undefined ? label : `${label}/${action}`;
+}
+/** Render a {@link RunDiff} as plain lines. */
+export function renderRunDiff(diff) {
+    const lines = ['', 'RUN-OVER-RUN DIFF'];
+    lines.push(`  compared: ${diff.compared} case(s)`);
+    for (const warning of diff.warnings)
+        lines.push(`  ! ${warning}`);
+    const list = (title, entries) => {
+        if (entries.length === 0)
+            return;
+        lines.push(`  ${title} (${entries.length}):`);
+        for (const entry of entries)
+            lines.push(`    ${entry.id}: ${entry.before} → ${entry.after}`);
+    };
+    list('REGRESSED', diff.regressed);
+    list('fixed', diff.fixed);
+    list('reclassified', diff.reclassified);
+    if (diff.metricDeltas.length > 0) {
+        lines.push('  metric deltas:');
+        for (const delta of diff.metricDeltas) {
+            const format = (value) => value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
+            const change = delta.delta === null
+                ? 'n/a'
+                : `${delta.delta >= 0 ? '+' : ''}${(delta.delta * 100).toFixed(1)}pp`;
+            lines.push(`    ${delta.name}: ${format(delta.before)} → ${format(delta.after)} (${change})`);
+        }
+    }
+    if (diff.regressed.length === 0 &&
+        diff.fixed.length === 0 &&
+        diff.reclassified.length === 0 &&
+        diff.metricDeltas.every((delta) => delta.delta === 0)) {
+        lines.push('  no change.');
+    }
+    return lines;
+}
+/** Does this suite declare a sweep? Small helper so the command reads declaratively. */
+export function suiteSweep(suite) {
+    return suite.sweep;
+}
+//# sourceMappingURL=evalCompare.js.map

package/dist/evalCompare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalCompare.js","sourceRoot":"","sources":["../src/evalCompare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAqC9C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,IAAI,KAAK,GAA2D,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,KAAyB,CAAC;QAC9B,IAAI,MAAM,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/E,KAAK;YACL,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACvB,IAAO,EACP,SAAkC;IAElC,MAAM,GAAG,GAA4B,EAAE,GAAG,IAAI,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAQD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,qBAAqB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,MAAgB,EAAU,EAAE,CACtD,KAAK,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;IAE/F,KAAK,CAAC,IAAI,CACR,GAAG,CACD,WAAW,EACX,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACrB,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC;QACxB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CACvD,CACF,CACF,CAAC;IACF,KAAK,CAAC,IAAI,CACR,GAAG,CACD,YAAY,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,QAAQ,CAAC;QACzD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,gGAAgG;IAChG,uFAAuF;IACvF,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,GAAG,CACD,IAAI,EACJ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAClE,CACF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CACR,GAAG,CACD,OAAO,GAAG,EAAE,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAC5D,CACF,CAAC;QACJ,CAAC;QACD,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CACzC,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CACnE,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CACR,KAAK,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,mBAAmB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;6BAC6B;AAC7B,SAAS,eAAe,CACtB,MAAwB,EACxB,UAAkB,EAClB,GAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACzF,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,KAAK,GAAgC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClG,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAgCD;gBACgB;AAChB,SAAS,OAAO,CAAC,MAAyC;IACxD,OAAO,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,EAAE,CAAC;AACxF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAwB,EAAE,KAAuB;IACxE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,QAAQ,IAAI,CAAC,CAAC;QAEd,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,UAAU,CAAC,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,sBAAsB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,sBAAsB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,WAAW,KAAK,UAAU,IAAI,CAAC,WAAW,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,CAAC;YAC9E,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,cAAc,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,YAAY;YAAE,SAAS;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC;QAC/C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,WAAW,KAAK,IAAI,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,WAAW;SACrF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CACX,6CAA6C,YAAY,CAAC,MAAM,yBAAyB;YACvF,GAAG,WAAW,CAAC,MAAM,4CAA4C,QAAQ,cAAc;YACvF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,KAAK;QACL,YAAY;QACZ,YAAY;QACZ,YAAY;QACZ,WAAW;QACX,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,cAA2E;IAE3E,IAAI,CAAC,cAAc;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,IAAI,GAAG,CAAC;IAChD,MAAM,MAAM,GAAG,cAAc,CAAC,YAAY,CAAC;IAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;AAC7D,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAa;IACzC,MAAM,KAAK,GAAa,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,UAAU,CAAC,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,OAAuB,EAAQ,EAAE;QAC5D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAExC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,CAAC,KAAoB,EAAU,EAAE,CAC9C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1D,MAAM,MAAM,GACV,KAAK,CAAC,KAAK,KAAK,IAAI;gBAClB,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,IACE,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QAC9B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,EACrD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC"}

package/dist/evalRunner.d.ts

@@ -1,5 +1,6 @@
 import type { RunCellFn } from '#src/types.js';
-import type { EvalSuite, EvalSuiteSummary, JudgeFn, RunConversationFn } from '#src/evalTypes.js';
+import type { EvalCaseResult, EvalSuite, EvalSuiteSummary, JudgeFn, RunClassifyFn, RunConversationFn } from '#src/evalTypes.js';
+import type { ClassifiedCell } from '#src/classificationTypes.js';
 /** Options for {@link runEvalSuite}. */
 export interface RunEvalSuiteOptions {
     /**
@@ -35,9 +36,24 @@ export interface RunEvalSuiteOptions {
      * reason (the runner degrades safely rather than throwing). The judge is orthogonal to identity:
      * it grades each cell's answer regardless of which identity produced it. */
     judge?: JudgeFn;
-    /** Max in-flight cells — reuses BATCH-1's `runBatchMatrix` pool (`DEFAULT_CONCURRENCY` when
-     * omitted); no second concurrency mechanism is introduced for eval. */
+    /** Max in-flight cells — reuses BATCH-1's `runBatchMatrix` pool (`DEFAULT_CELL_CONCURRENCY`,
+     * i.e. serial, when omitted); no second concurrency mechanism is introduced for eval. */
     concurrency?: number;
+    /**
+     * BATCH-25 Half B SEAM — an injected CLASSIFICATION target. When provided (and the suite declares
+     * a `classification:` block), every case is dispatched through it INSTEAD of the agent runners:
+     * the target returns the label/action per round directly, plus the model-call count that makes
+     * `model_free` enforceable.
+     *
+     * Half B's `rater` target supplies it (`buildRaterClassifier`, #src/raterTarget.js) by driving
+     * the approvals rating prompt + decision mapping at a declared rung. REQUIRED for a `rater`
+     * suite — the guard at the top of {@link runEvalSuite} rejects the run without it rather than
+     * letting the cases fall through to the agent.
+     *
+     * When absent, a classifier suite reads its label/action out of the ordinary agent answer via the
+     * suite's declared extractors, which is what makes the facility usable on its own today.
+     */
+    classify?: RunClassifyFn;
 }
 /**
  * Run every (case × identity) cell of the suite through the SUT ({@link RunCellFn}, pooled via
@@ -56,6 +72,15 @@ export interface RunEvalSuiteOptions {
  * single-turn unit keeps the proven `runCell` + {@link gradeUnit} path byte-for-byte.
  */
 export declare function runEvalSuite(suite: EvalSuite, options: RunEvalSuiteOptions): Promise<EvalSuiteSummary>;
+/**
+ * BATCH-25 — reduce the graded results to the shape the matrices and metrics read.
+ *
+ * `scored` is the load-bearing field: a cell whose SUT never ran has NO place in any matrix cell or
+ * any metric denominator, and is instead counted as `excluded` and said out loud. Silently treating
+ * it as a wrong answer would inflate the error rate; silently dropping it would inflate coverage.
+ * Both are how a classifier report comes to claim more than it measured.
+ */
+export declare function toClassifiedCells(results: EvalCaseResult[]): ClassifiedCell[];
 /** The three-way process exit code for a completed `gth eval` run. See {@link classifyEvalExit}. */
 export type EvalExitCode = 0 | 1 | 2;
 /**
@@ -74,5 +99,11 @@ export type EvalExitCode = 0 | 1 | 2;
  * Classification is anchored on `sutOk`, not the verdict: a cell that ran (`sutOk === true`) but
  * whose judge errored (or whose answer failed a check) is a real result → exit `1`, never `2`. A
  * *mix* of `sutOk:false` and `sutOk:true` cells therefore yields `1`.
+ *
+ * BATCH-25 adds ONE further way to reach `1`: a breached `gate: fail` metric threshold. That is a
+ * product signal of exactly the same kind — and it is the one that a per-case pass/fail sweep cannot
+ * express, because a classifier corpus can be entirely within per-case tolerance while its
+ * false-approve rate is unacceptable. Per-case verdicts answer "did each case behave"; a gated
+ * metric answers "is the aggregate shippable", and only the second gates a release.
  */
 export declare function classifyEvalExit(summary: EvalSuiteSummary): EvalExitCode;