@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.26

package/dist/classificationRender.js

@@ -0,0 +1,96 @@
+import { formatTally } from '#src/metrics.js';
+/** Render the whole report: coverage, matrices, metrics, warnings, gate verdict. */
+export function renderClassificationReport(report, options = {}) {
+    const lines = [];
+    lines.push('');
+    lines.push('CLASSIFICATION');
+    // Coverage FIRST, before any number that depends on it. "no silent caps" is not a footnote.
+    lines.push(`  coverage: ${report.coverage.scored}/${report.coverage.total} cell(s) classified` +
+        (report.coverage.excluded > 0 ? `, ${report.coverage.excluded} excluded` : ''));
+    for (const warning of report.warnings)
+        lines.push(`  ! ${warning}`);
+    lines.push('');
+    lines.push(...renderConfusionMatrix(report.labelMatrix, 'label'));
+    if (report.actionMatrix) {
+        lines.push('');
+        lines.push(...renderConfusionMatrix(report.actionMatrix, 'action'));
+    }
+    // Per-tag matrices only when there is more than one family — with a single tag the per-tag matrix
+    // is the overall one, and printing it twice is noise, not information — and never in compact mode.
+    if (report.tags.length > 1 && !options.compact) {
+        for (const tag of report.tags) {
+            const matrix = report.labelMatrixByTag[tag];
+            if (!matrix)
+                continue;
+            lines.push('');
+            lines.push(...renderConfusionMatrix(matrix, `label · tag "${tag}"`));
+        }
+    }
+    if (options.compact && report.tags.length > 1) {
+        // Say what was withheld. A renderer that quietly drops a section is a silent cap, which is the
+        // one thing this facility may not do — even about its own output.
+        lines.push('');
+        lines.push(`  (per-tag confusion matrices for ${report.tags.length} tag(s) omitted in a sweep cell — ` +
+            "they are in this cell's results.json; per-tag metric rows are below and in the " +
+            'comparison table.)');
+    }
+    if (report.metrics.length > 0) {
+        lines.push('');
+        lines.push('METRICS');
+        for (const metric of report.metrics)
+            lines.push(...renderMetric(metric, report.tags));
+    }
+    if (report.gateFailures.length > 0) {
+        lines.push('');
+        lines.push(`METRIC GATE FAILED: ${report.gateFailures.join(', ')}`);
+    }
+    return lines;
+}
+/** Render one confusion matrix as a fixed-width grid: rows = expected, columns = actual. */
+export function renderConfusionMatrix(matrix, title) {
+    const lines = [];
+    lines.push(`  confusion (${title}) — rows = expected, cols = actual`);
+    const rowLabelWidth = Math.max(...matrix.rows.map((row) => row.length), 'expected'.length);
+    const columnWidths = matrix.columns.map((column) => Math.max(column.length, ...matrix.rows.map((row) => String(matrix.counts[row][column] ?? 0).length)));
+    const header = matrix.columns
+        .map((column, index) => column.padStart(columnWidths[index] + 2))
+        .join('');
+    lines.push(`    ${''.padEnd(rowLabelWidth)}${header}`);
+    for (const row of matrix.rows) {
+        const cells = matrix.columns
+            .map((column, index) => String(matrix.counts[row][column] ?? 0).padStart(columnWidths[index] + 2))
+            .join('');
+        lines.push(`    ${row.padEnd(rowLabelWidth)}${cells}`);
+    }
+    lines.push(`    counted ${matrix.counted}` +
+        (matrix.excluded > 0 ? `, EXCLUDED ${matrix.excluded} (not classified)` : ''));
+    return lines;
+}
+/** Render one metric: headline value, gate verdict, per-tag sub-scores, then its warnings. */
+export function renderMetric(metric, tags) {
+    const lines = [];
+    // The threshold's UNIT is always on the line, passing or failing. `[gate ok]` alone left a reader
+    // unable to tell whether the bar was two cases or 2% — and the count form exists precisely
+    // because those are different rules that drift apart as the corpus grows.
+    const gateSuffix = metric.gate
+        ? metric.gate.passed
+            ? ` [gate ok: ${metric.gate.summary}]`
+            : ` [GATE ${metric.gate.mode === 'fail' ? 'FAILED' : 'breached (report-only)'}: ${metric.gate.reason ?? `threshold ${metric.gate.summary} breached`}]`
+        : '';
+    lines.push(`  ${metric.name}: ${formatTally(metric.overall)}${gateSuffix}`);
+    if (metric.description)
+        lines.push(`    ${metric.description}`);
+    // Per-tag ALWAYS, not only when interesting: the 48.9%-overall / 0-of-3-on-injection result is
+    // exactly the one a "print it if it looks bad" rule would have hidden, since it looked fine.
+    for (const tag of tags) {
+        const tally = metric.byTag[tag];
+        if (!tally)
+            continue;
+        lines.push(`    ${tag}: ${formatTally(tally)}`);
+    }
+    lines.push(`    coverage: denominator ${metric.coverage.denominator}/${metric.coverage.total} case(s)`);
+    for (const warning of metric.warnings)
+        lines.push(`    ! ${warning}`);
+    return lines;
+}
+//# sourceMappingURL=classificationRender.js.map

package/dist/classificationRender.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classificationRender.js","sourceRoot":"","sources":["../src/classificationRender.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAqB9C,oFAAoF;AACpF,MAAM,UAAU,0BAA0B,CACxC,MAAgC,EAChC,OAAO,GAAgC,EAAE;IAEzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7B,4FAA4F;IAC5F,KAAK,CAAC,IAAI,CACR,eAAe,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,qBAAqB;QACjF,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,QAAQ,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CACjF,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;IAEpE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IAClE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,kGAAkG;IAClG,mGAAmG;IACnG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,GAAG,GAAG,CAAC,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,+FAA+F;QAC/F,kEAAkE;QAClE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,qCAAqC,MAAM,CAAC,IAAI,CAAC,MAAM,oCAAoC;YACzF,iFAAiF;YACjF,oBAAoB,CACvB,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,qBAAqB,CAAC,MAA2B,EAAE,KAAa;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,oCAAoC,CAAC,CAAC;IAEtE,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3F,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACjD,IAAI,CAAC,GAAG,CACN,MAAM,CAAC,MAAM,EACb,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAC5E,CACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO;SAC1B,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAChE,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;IAEvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO;aACzB,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CACrB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAC1E;aACA,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,eAAe,MAAM,CAAC,OAAO,EAAE;QAC7B,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,QAAQ,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAChF,CAAC;IACF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,YAAY,CAAC,MAAwB,EAAE,IAAc;IACnE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,kGAAkG;IAClG,2FAA2F;IAC3F,0EAA0E;IAC1E,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI;QAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM;YAClB,CAAC,CAAC,cAAc,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG;YACtC,CAAC,CAAC,UAAU,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,KACzE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,aAAa,MAAM,CAAC,IAAI,CAAC,OAAO,WACxD,GAAG;QACP,CAAC,CAAC,EAAE,CAAC;IACP,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,KAAK,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,UAAU,EAAE,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,WAAW;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAEhE,+FAA+F;IAC/F,6FAA6F;IAC7F,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,6BAA6B,MAAM,CAAC,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAC5F,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,EAAE,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/classificationReport.d.ts

@@ -0,0 +1,11 @@
+import type { ClassifiedCell, EvalClassificationReport, EvalClassificationSpec, EvalMetricSpec } from '#src/classificationTypes.js';
+/**
+ * BATCH-25 — assemble the suite-level classification report: the confusion matrices (overall and
+ * per tag), every declared metric (overall and per tag), the corpus-wide coverage, and the list of
+ * `gate: fail` metrics that were breached.
+ *
+ * Kept separate from both the extractor ({@link ./classification.js}) and the metric engine
+ * ({@link ./metrics.js}) so neither imports the other, and so the whole aggregation is exercisable
+ * from a plain array of {@link ClassifiedCell}s with no runner, no I/O, and no model.
+ */
+export declare function buildClassificationReport(spec: EvalClassificationSpec, metricSpecs: EvalMetricSpec[], cells: ClassifiedCell[]): EvalClassificationReport;

package/dist/classificationReport.js

@@ -0,0 +1,60 @@
+import { buildConfusionMatrix, collectTags } from '#src/classification.js';
+import { computeMetric } from '#src/metrics.js';
+import { UNRECOGNIZED_LABEL } from '#src/classificationTypes.js';
+/**
+ * BATCH-25 — assemble the suite-level classification report: the confusion matrices (overall and
+ * per tag), every declared metric (overall and per tag), the corpus-wide coverage, and the list of
+ * `gate: fail` metrics that were breached.
+ *
+ * Kept separate from both the extractor ({@link ./classification.js}) and the metric engine
+ * ({@link ./metrics.js}) so neither imports the other, and so the whole aggregation is exercisable
+ * from a plain array of {@link ClassifiedCell}s with no runner, no I/O, and no model.
+ */
+export function buildClassificationReport(spec, metricSpecs, cells) {
+    const tags = collectTags(cells);
+    const total = cells.length;
+    const scored = cells.filter((cell) => cell.scored);
+    const excluded = total - scored.length;
+    const labelMatrix = buildConfusionMatrix(cells, 'label', spec.labels);
+    const actionMatrix = spec.actions.length > 0 ? buildConfusionMatrix(cells, 'action', spec.actions) : undefined;
+    // Per-tag matrices are built from the tag's OWN cells, so a family's matrix reads on its own —
+    // the 0/3-on-prompt-injection result that a 48.9% aggregate hid.
+    const labelMatrixByTag = {};
+    for (const tag of tags) {
+        labelMatrixByTag[tag] = buildConfusionMatrix(cells.filter((cell) => cell.tags.includes(tag)), 'label', spec.labels);
+    }
+    const metrics = metricSpecs.map((metricSpec) => computeMetric(metricSpec, cells, tags));
+    // Report-level warnings: everything that bounds what these numbers cover, stated once at the top
+    // rather than left for a reader to infer from a denominator. "No silent caps" is not a per-metric
+    // courtesy — it is the report's contract.
+    const warnings = [];
+    if (excluded > 0) {
+        warnings.push(`${excluded}/${total} cell(s) produced no classification (the SUT did not run, or the ` +
+            'classifier failed) and are excluded from every matrix and metric below. Coverage is ' +
+            `${scored.length}/${total}, NOT ${total}/${total}.`);
+    }
+    const unrecognized = scored.filter((cell) => cell.actualLabel === UNRECOGNIZED_LABEL || cell.actualAction === UNRECOGNIZED_LABEL);
+    if (unrecognized.length > 0) {
+        warnings.push(`${unrecognized.length} cell(s) produced output matching no declared value and are counted ` +
+            `under "${UNRECOGNIZED_LABEL}" (e.g. ${unrecognized
+                .slice(0, 3)
+                .map((cell) => cell.id)
+                .join(', ')}). They are scored, not dropped — an uninterpretable verdict is a finding.`);
+    }
+    const gateFailures = metrics
+        .filter((metric) => metric.gate && metric.gate.mode === 'fail' && !metric.gate.passed)
+        .map((metric) => metric.name);
+    return {
+        labels: spec.labels,
+        actions: spec.actions,
+        tags,
+        coverage: { total, scored: scored.length, excluded },
+        labelMatrix,
+        actionMatrix,
+        labelMatrixByTag,
+        metrics,
+        warnings,
+        gateFailures,
+    };
+}
+//# sourceMappingURL=classificationReport.js.map

package/dist/classificationReport.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classificationReport.js","sourceRoot":"","sources":["../src/classificationReport.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAOhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAEjE;;;;;;;;GAQG;AACH,MAAM,UAAU,yBAAyB,CACvC,IAA4B,EAC5B,WAA6B,EAC7B,KAAuB;IAEvB,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAEvC,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,MAAM,YAAY,GAChB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5F,+FAA+F;IAC/F,iEAAiE;IACjE,MAAM,gBAAgB,GAA4D,EAAE,CAAC;IACrF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,gBAAgB,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAC1C,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAC/C,OAAO,EACP,IAAI,CAAC,MAAM,CACZ,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;IAExF,iGAAiG;IACjG,kGAAkG;IAClG,0CAA0C;IAC1C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CACX,GAAG,QAAQ,IAAI,KAAK,mEAAmE;YACrF,sFAAsF;YACtF,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,KAAK,GAAG,CACtD,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAChC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,kBAAkB,IAAI,IAAI,CAAC,YAAY,KAAK,kBAAkB,CAC9F,CAAC;IACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CACX,GAAG,YAAY,CAAC,MAAM,sEAAsE;YAC1F,UAAU,kBAAkB,WAAW,YAAY;iBAChD,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;iBACtB,IAAI,CAAC,IAAI,CAAC,4EAA4E,CAC5F,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,OAAO;SACzB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;SACrF,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEhC,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,IAAI;QACJ,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE;QACpD,WAAW;QACX,YAAY;QACZ,gBAAgB;QAChB,OAAO;QACP,QAAQ;QACR,YAAY;KACb,CAAC;AACJ,CAAC"}

package/dist/classificationTypes.d.ts

@@ -0,0 +1,311 @@
+/**
+ * @packageDocumentation
+ * BATCH-25 — the shapes for a CLASSIFIER eval: a suite-declared label/action enum, the extractors
+ * that turn a SUT answer into a label, the confusion matrix, and declared aggregate metrics.
+ *
+ * Deliberately its own file rather than more weight in {@link ./evalTypes.js}: eval's per-case
+ * PASS/FAIL shapes and a classifier's corpus-wide *distribution* shapes answer different questions,
+ * and only the latter needs the anti-blind-metric machinery below.
+ *
+ * ## The lesson this file is shaped by
+ *
+ * The QA-5 throwaway harness declared an `over-rejection` metric whose denominator was only the
+ * `safe`-labelled cases. It read a clean **0/10 at both strictness settings** while that setting was
+ * simultaneously rejecting seven routine commands back to the model. A perfect score, reported for a
+ * setting that had made behaviour materially worse, and then trusted.
+ *
+ * So, structurally, in this file:
+ * - a metric's denominator is **the whole scored corpus unless the author says otherwise**
+ *   ({@link EvalMetricSpec.over} is optional and its absence means "everything");
+ * - every metric carries {@link EvalMetricResult.coverage}, and any shortfall — a declared subset
+ *   denominator, cases excluded because the SUT failed, an empty denominator, or numerator cases
+ *   sitting OUTSIDE the denominator — becomes a {@link EvalMetricResult.warnings} entry;
+ * - an unrecognized label is a real, visible bucket ({@link UNRECOGNIZED_LABEL}) rather than a
+ *   dropped row, so the confusion matrix always accounts for every scored cell.
+ *
+ * A blind metric is worse than no metric, because it is trusted.
+ */
+/** The bucket a classification lands in when the SUT produced output but it matched no declared
+ * label/action. It is a REAL row/column of the confusion matrix, never a dropped case — an
+ * unparseable verdict is a finding, and silently discarding it is how a matrix comes to read
+ * "covered everything" when it didn't. Parenthesized so it can never collide with a declared enum
+ * value (which must match `/^[\w.-]+$/`). */
+export declare const UNRECOGNIZED_LABEL = "(unrecognized)";
+/** The axis entry for "this case declares no expectation on this dimension" — e.g. a corpus case
+ * that asserts an action but no label. Like {@link UNRECOGNIZED_LABEL} it is a visible bucket.
+ * Parenthesized for the same collision-proof reason. */
+export declare const NO_EXPECTATION = "(none)";
+/**
+ * How a label/action is read out of a cell's answer. Deliberately tiny and TOTAL — every extractor
+ * either yields a declared enum value or {@link UNRECOGNIZED_LABEL}; none of them guesses.
+ *
+ * - `answer` (default) — the trimmed answer, compared case-insensitively against the declared enum.
+ *   Suited to a suite whose prompt says "reply with exactly one of: …".
+ * - `json_path` — resolve a minimal dot/`[index]` path (the same resolver `json_path` assertions
+ *   use) against the answer parsed as JSON, then match its stringified value against the enum.
+ *   Suited to a structured-output classifier.
+ *
+ * There is deliberately NO fuzzy/substring mode. A heuristic that finds `safe` inside "this is not
+ * safe" is precisely the class of silent misreading this facility exists to eliminate.
+ */
+export type ClassificationExtractor = {
+    kind: 'answer';
+} | {
+    kind: 'json_path';
+    path: string;
+};
+/**
+ * A suite's `classification:` declaration — the enum that gives the confusion matrix its axes, plus
+ * how to read a value out of the SUT's answer.
+ *
+ * `actions` is empty when the suite classifies labels only. The two dimensions are separate because
+ * the QA-5 corpus asserts two different things about the same case: **the label the model returned**
+ * (diagnostic) and **the action the gate took** (what a user actually experiences). They diverge by
+ * design — deterministic preflights rewrite the verdict independently of the model — so scoring
+ * labels alone overstates the model and understates the gate.
+ */
+export interface EvalClassificationSpec {
+    /** The declared label enum (≥1 value). Gives the matrix its axes and validates every
+     * `expect_label` and every metric literal at parse time. */
+    labels: string[];
+    /** The declared action enum, or `[]` when the suite asserts labels only. */
+    actions: string[];
+    /** How the ACTUAL label is read from a cell's answer. */
+    labelFrom: ClassificationExtractor;
+    /** How the ACTUAL action is read. `undefined` = the suite declares no action dimension; an
+     * `expect_action` assertion is then a parse error rather than a silently ungradeable one. */
+    actionFrom?: ClassificationExtractor;
+}
+/** One field a metric predicate can read. `expected.*` is what the CORPUS declares; `actual.*` is
+ * what the SUT produced. The prose in a corpus plan says "label" for the corpus label and "action"
+ * for the actual action taken, and that ambiguity is exactly what produces a silently-wrong metric —
+ * so the surface forces the distinction to be written down. */
+export type MetricField = 'expected.label' | 'expected.action' | 'actual.label' | 'actual.action';
+/** The literal `none` in a predicate — matches a cell where that field is absent. */
+export declare const METRIC_NONE_LITERAL = "none";
+/**
+ * One comparison inside a metric's predicate list. Predicates in a list are **ANDed**; there is no
+ * `or`, no nesting and no parentheses.
+ *
+ * That is a deliberate limit, not an oversight. Every metric the approvals corpus plan declares —
+ * `false_approve`, `false_halt`, `over_escalation`, `outcome_accuracy`, `exfil_recall`,
+ * `floor_recall` — is a conjunction of at most two comparisons. A general expression grammar would
+ * add a parser to the ONE component where a bug yields wrong numbers that look right, which is the
+ * failure this whole file is built to prevent. If a corpus ever needs disjunction, that is a new
+ * node, not a silent generalisation.
+ */
+export type MetricPredicate = 
+/** `<field> == <value>` / `<field> != <value>`, where the right-hand side is a literal. */
+{
+    kind: 'compare';
+    field: MetricField;
+    negated: boolean;
+    value: string;
+}
+/** `<field> == <field>` / `<field> != <field>` — how `outcome_accuracy` is written
+ * (`actual.label == expected.label`). */
+ | {
+    kind: 'compareField';
+    field: MetricField;
+    negated: boolean;
+    other: MetricField;
+}
+/** `<field> in [a, b]` / `<field> not in [a, b]`. */
+ | {
+    kind: 'in';
+    field: MetricField;
+    negated: boolean;
+    values: string[];
+}
+/** `has_tag(x)` / `not has_tag(x)`. */
+ | {
+    kind: 'tag';
+    negated: boolean;
+    tag: string;
+};
+/**
+ * A suite-declared aggregate metric over the corpus.
+ *
+ * `over` (the denominator) is OPTIONAL and its absence means **the whole scored corpus**. That
+ * default is the load-bearing part of this type: an author who does not think about the denominator
+ * gets the corpus-wide one, and an author who narrows it gets a warning saying how much of the
+ * corpus their number can no longer see.
+ */
+export interface EvalMetricSpec {
+    name: string;
+    /** The numerator predicate list (ANDed). Evaluated only over cells already in the denominator. */
+    where: MetricPredicate[];
+    /** The denominator predicate list (ANDed). Absent = every scored cell — corpus-wide by
+     * construction. */
+    over?: MetricPredicate[];
+    /** FRACTION gate: fail when the value is strictly GREATER than this (0..1). */
+    max?: number;
+    /** FRACTION gate: fail when the value is strictly LESS than this (0..1) — or when the denominator
+     * is empty (a recall metric that measured nothing has not met its floor; it has measured
+     * nothing). */
+    min?: number;
+    /**
+     * COUNT gate: fail when the NUMERATOR exceeds this many cases.
+     *
+     * Not sugar for {@link max}. A target like "at most 2 of the 22 cases in this family" is an
+     * absolute count against an honest denominator, and expressing it as a fraction has two failures
+     * the fraction form cannot avoid:
+     *
+     * - the author must compute `2/22 = 0.0909` by hand from the current corpus size, and
+     * - **it silently drifts as the corpus grows.** Add ten cases and the gate quietly tightens or
+     *   loosens — no edit, no warning, the number still plausible while its meaning has moved.
+     *
+     * That second one is the same species as the blind denominator the rest of this file guards
+     * against: a number that stays believable while what it measures changes underneath it. A count
+     * gate is invariant to corpus size by construction.
+     *
+     * A count gate also reads more honestly wherever the target genuinely IS zero cases —
+     * `max_count: 0` says "not one case may do this", where `max: 0` says it in a unit that happens
+     * to coincide at zero.
+     */
+    maxCount?: number;
+    /** COUNT gate: fail when the NUMERATOR is below this many cases. Unlike {@link min}, this needs
+     * no empty-denominator special case — an empty denominator yields a numerator of 0, which is
+     * simply below any positive floor. */
+    minCount?: number;
+    /** `fail` (default when a threshold is declared) — a tripped gate fails the run (exit 1).
+     * `report` — the threshold is computed and printed but never changes the exit code. */
+    gate: 'fail' | 'report';
+    /** Optional human note carried into the output, for a metric whose meaning is not its name. */
+    description?: string;
+}
+/** One metric's numerator/denominator/value. `value` is `null` — never `0` — when the denominator is
+ * empty, because "0% of nothing" and "0% of the corpus" are opposite findings and a metric that
+ * silently reports the former as the latter is the exact bug this facility exists to catch. */
+export interface EvalMetricTally {
+    numerator: number;
+    denominator: number;
+    value: number | null;
+}
+/** How much of the corpus a metric could actually see. Always emitted, never conditional — point 9
+ * ("no silent caps") and the anti-blind-metric rule are the same requirement, so they are the same
+ * field. */
+export interface EvalMetricCoverage {
+    /** Every cell in the suite. */
+    total: number;
+    /** Cells that produced a gradeable classification (the SUT ran). */
+    scored: number;
+    /** `total - scored` — cells that could not be classified at all. */
+    excluded: number;
+    /** How many scored cells this metric's own denominator selected. */
+    denominator: number;
+}
+/** One metric's computed result, overall and per tag. */
+export interface EvalMetricResult {
+    name: string;
+    description?: string;
+    overall: EvalMetricTally;
+    /** Per-family sub-scores. An aggregate hides adversarial collapse: the first QA-5 baseline scored
+     * 48.9% overall while scoring 0/3 on prompt injection, and a single blended number would have
+     * shipped that. Keyed by tag, in the suite's tag order. */
+    byTag: Record<string, EvalMetricTally>;
+    coverage: EvalMetricCoverage;
+    /** Every way this metric's denominator falls short of the whole corpus. Empty is the good case. */
+    warnings: string[];
+    /** Present when the metric declares any threshold. */
+    gate?: {
+        /** Which unit the thresholds are in. A consumer must never have to infer whether `2` meant two
+         * cases or 200% — the two forms are mutually exclusive per metric, and this names which. */
+        kind: 'fraction' | 'count';
+        max?: number;
+        min?: number;
+        maxCount?: number;
+        minCount?: number;
+        mode: 'fail' | 'report';
+        /** `false` = the threshold was breached. A breached `fail` gate fails the run. */
+        passed: boolean;
+        /** Why it failed; empty when it passed. Always states the unit. */
+        reason?: string;
+        /** The threshold as a human string, unit included (`≤ 2 case(s)`, `≤ 5.0%`). Rendered whether
+         * the gate passed or failed, so a passing gate is as legible as a failing one. */
+        summary: string;
+    };
+}
+/**
+ * A confusion matrix: rows = what the corpus expected, columns = what the SUT actually produced.
+ *
+ * The primary artifact of a classifier eval, because **which way it is wrong is the whole signal**:
+ * on the approvals scale (`safe` · `destructive` · `catastrophic` · `attack`) an `attack` graded
+ * `destructive` means a prompt instead of a halt, while a `destructive` graded `safe` is a security
+ * incident. A single accuracy percentage cannot tell those apart.
+ */
+export interface EvalConfusionMatrix {
+    /** Which dimension this matrix is over. */
+    dimension: 'label' | 'action';
+    /** Row axis (expected): the declared enum, plus {@link NO_EXPECTATION} when some cell declares
+     * none. */
+    rows: string[];
+    /** Column axis (actual): the declared enum, plus {@link UNRECOGNIZED_LABEL} when some cell
+     * produced an unmatched value, plus {@link NO_EXPECTATION} when some cell produced nothing. */
+    columns: string[];
+    /** `counts[expectedRow][actualColumn]`. Every row/column key is present (zeros included) so a
+     * renderer never has to guess an axis. */
+    counts: Record<string, Record<string, number>>;
+    /** Cells counted here. `counted + excluded === ` the suite's cell total — asserted in the unit
+     * suite, because a matrix that silently drops errored cells reads as "covered everything". */
+    counted: number;
+    /** Cells NOT counted: the SUT did not run, so there is nothing to place. */
+    excluded: number;
+}
+/**
+ * The suite-level classification report — the classifier eval's answer, attached to
+ * {@link ./evalTypes.js EvalSuiteSummary}. Absent entirely for a suite that declares no
+ * `classification:` block, so a #405-era suite's `results.json` is unchanged.
+ */
+export interface EvalClassificationReport {
+    labels: string[];
+    actions: string[];
+    /** Every tag any case declared, sorted — the per-tag sub-score axis. */
+    tags: string[];
+    /** Corpus-wide coverage. `excluded > 0` means some cells could not be classified at all. */
+    coverage: {
+        total: number;
+        scored: number;
+        excluded: number;
+    };
+    labelMatrix: EvalConfusionMatrix;
+    /** Present only when the suite declares an action dimension. */
+    actionMatrix?: EvalConfusionMatrix;
+    /** Per-tag label matrices, keyed by tag — an adversarial family's matrix read on its own. */
+    labelMatrixByTag: Record<string, EvalConfusionMatrix>;
+    metrics: EvalMetricResult[];
+    /** Report-level warnings (as opposed to per-metric ones): excluded cells, unrecognized outputs,
+     * and anything else that bounds what these numbers cover. */
+    warnings: string[];
+    /** Names of metrics whose `gate: fail` threshold was breached. Non-empty ⇒ the run fails. */
+    gateFailures: string[];
+}
+/**
+ * One graded cell reduced to what the matrices and the metrics need. Built by
+ * {@link ../evalRunner.js} from the graded results; its own tiny shape so both consumers read the
+ * same thing and the aggregation is unit-testable without constructing whole `EvalCaseResult`s.
+ */
+export interface ClassifiedCell {
+    id: string;
+    tags: string[];
+    expectedLabel?: string;
+    expectedAction?: string;
+    actualLabel?: string;
+    actualAction?: string;
+    /** `false` = the SUT did not run / the classifier failed, so this cell has no place in any matrix
+     * or denominator. It is counted as `excluded` and reported, never as a wrong answer. */
+    scored: boolean;
+}
+/** One cell's classification, as recorded on its {@link ./evalTypes.js EvalCaseResult}. */
+export interface EvalCaseClassification {
+    expectedLabel?: string;
+    expectedAction?: string;
+    actualLabel?: string;
+    actualAction?: string;
+    /** The raw text the extractors read, kept so an `(unrecognized)` result is diagnosable without
+     * re-running. Omitted when it is identical to the cell's `answer`. */
+    raw?: string;
+    /** How many model calls this classification cost. Present only on the injected-classifier
+     * (Half B) path; the answer-extraction path always costs the cell's own single run. */
+    modelCalls?: number;
+}

package/dist/classificationTypes.js

@@ -0,0 +1,40 @@
+/**
+ * @packageDocumentation
+ * BATCH-25 — the shapes for a CLASSIFIER eval: a suite-declared label/action enum, the extractors
+ * that turn a SUT answer into a label, the confusion matrix, and declared aggregate metrics.
+ *
+ * Deliberately its own file rather than more weight in {@link ./evalTypes.js}: eval's per-case
+ * PASS/FAIL shapes and a classifier's corpus-wide *distribution* shapes answer different questions,
+ * and only the latter needs the anti-blind-metric machinery below.
+ *
+ * ## The lesson this file is shaped by
+ *
+ * The QA-5 throwaway harness declared an `over-rejection` metric whose denominator was only the
+ * `safe`-labelled cases. It read a clean **0/10 at both strictness settings** while that setting was
+ * simultaneously rejecting seven routine commands back to the model. A perfect score, reported for a
+ * setting that had made behaviour materially worse, and then trusted.
+ *
+ * So, structurally, in this file:
+ * - a metric's denominator is **the whole scored corpus unless the author says otherwise**
+ *   ({@link EvalMetricSpec.over} is optional and its absence means "everything");
+ * - every metric carries {@link EvalMetricResult.coverage}, and any shortfall — a declared subset
+ *   denominator, cases excluded because the SUT failed, an empty denominator, or numerator cases
+ *   sitting OUTSIDE the denominator — becomes a {@link EvalMetricResult.warnings} entry;
+ * - an unrecognized label is a real, visible bucket ({@link UNRECOGNIZED_LABEL}) rather than a
+ *   dropped row, so the confusion matrix always accounts for every scored cell.
+ *
+ * A blind metric is worse than no metric, because it is trusted.
+ */
+/** The bucket a classification lands in when the SUT produced output but it matched no declared
+ * label/action. It is a REAL row/column of the confusion matrix, never a dropped case — an
+ * unparseable verdict is a finding, and silently discarding it is how a matrix comes to read
+ * "covered everything" when it didn't. Parenthesized so it can never collide with a declared enum
+ * value (which must match `/^[\w.-]+$/`). */
+export const UNRECOGNIZED_LABEL = '(unrecognized)';
+/** The axis entry for "this case declares no expectation on this dimension" — e.g. a corpus case
+ * that asserts an action but no label. Like {@link UNRECOGNIZED_LABEL} it is a visible bucket.
+ * Parenthesized for the same collision-proof reason. */
+export const NO_EXPECTATION = '(none)';
+/** The literal `none` in a predicate — matches a cell where that field is absent. */
+export const METRIC_NONE_LITERAL = 'none';
+//# sourceMappingURL=classificationTypes.js.map

package/dist/classificationTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classificationTypes.js","sourceRoot":"","sources":["../src/classificationTypes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;;6CAI6C;AAC7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAEnD;;wDAEwD;AACxD,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC;AA8CvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC"}

package/dist/evalCompare.d.ts

@@ -0,0 +1,108 @@
+import type { EvalSuite, EvalSuiteSummary, EvalSweep } from '#src/evalTypes.js';
+/**
+ * BATCH-25 — the comparison layer: run the same corpus across a sweep of configurations and emit
+ * ONE comparison table, and diff a run against a previous one.
+ *
+ * ## Why a sweep is an OUTER loop and not a runner concept
+ *
+ * A sweep is "the same suite, a different config" — structurally identical to BATCH-19's
+ * multi-suite loop, and unrelated to grading. Threading it into `runEvalSuite` would have meant
+ * generalising the (case × identity) unit to (case × identity × sweep cell) and rewriting the
+ * `RunCellFn` resolution that #405's identity matrix depends on. Instead the command runs the suite
+ * once per cell and this module folds the N summaries into one table. The identity matrix is
+ * untouched, and the run-over-run diff falls out of the same code for free.
+ *
+ * ## Why one table and not N reports
+ *
+ * The decisive QA-5 result came from running one corpus at two settings and diffing them; N
+ * separate reports is the form in which that result is invisible. So the artifact is a table whose
+ * rows are metrics and whose columns are cells, plus the same for overall accuracy.
+ */
+/** One sweep cell: a name and the config overrides that produce it. */
+export interface SweepCell {
+    /** `axis=value` joined by ` · ` across axes — stable, and derived from the declared names. */
+    name: string;
+    /** A filename-safe form of {@link name} (`rung-auto-safe__model-flash`), used as an output-dir
+     * component. Built from parse-time-validated path-safe tokens, so it can neither traverse nor
+     * escape the output root. */
+    dirName: string;
+    /** The `model` override for this cell, when any axis value sets one. */
+    model?: string;
+    /** The merged plain-data config overrides for this cell. */
+    config: Record<string, unknown>;
+}
+/**
+ * Expand a {@link EvalSweep} into its cartesian product of cells, in declared axis order.
+ *
+ * A later axis wins on a conflicting key, and that is documented rather than defended against: two
+ * axes that set the same config key are describing the same knob twice, which the author should see
+ * in the cell name.
+ */
+export declare function expandSweep(sweep: EvalSweep): SweepCell[];
+/**
+ * Deep-merge plain data (the sweep's `config:` overrides) into a target.
+ *
+ * Objects merge recursively; arrays and scalars REPLACE. Replacing an array is the right default
+ * for config: an override that appended to `mcpServers` or `allowedTools` would silently keep the
+ * base value the author meant to displace, which is the harder bug to see.
+ *
+ * Prototype-polluting keys are skipped — a suite file is only semi-trusted input.
+ */
+export declare function deepMerge<T extends Record<string, unknown>>(base: T, overrides: Record<string, unknown>): T;
+/** One column of the comparison table: a cell name and the summary it produced. */
+export interface ComparisonColumn {
+    name: string;
+    summary: EvalSuiteSummary;
+}
+/**
+ * Render the cross-cell comparison table.
+ *
+ * Rows are the metrics the suite declares (plus pass rate), columns are the sweep cells. A metric
+ * that a cell could not compute renders `n/a`, never a blank and never a zero — the same rule the
+ * metric engine follows, for the same reason.
+ *
+ * Per-tag sub-scores get their own rows under each metric, because the whole point of running a
+ * sweep is to see which setting moved which family, and a blended per-cell number cannot show that.
+ */
+export declare function renderComparison(columns: ComparisonColumn[]): string[];
+/** One case whose verdict or classification moved between two runs. */
+export interface RunDiffEntry {
+    id: string;
+    before: string;
+    after: string;
+}
+/** The run-over-run diff. */
+export interface RunDiff {
+    /** Cases in both runs. */
+    compared: number;
+    /** Cases that went PASS → FAIL. The regression list. */
+    regressed: RunDiffEntry[];
+    /** Cases that went FAIL → PASS. */
+    fixed: RunDiffEntry[];
+    /** Cases whose actual label/action changed, verdict aside — a rating-prompt edit's real signal. */
+    reclassified: RunDiffEntry[];
+    /** Metric deltas, `after - before`, for metrics both runs computed. */
+    metricDeltas: {
+        name: string;
+        before: number | null;
+        after: number | null;
+        delta: number | null;
+    }[];
+    /** Ids in only one of the two runs — reported, so a shrunken corpus cannot read as "no change". */
+    onlyInBefore: string[];
+    onlyInAfter: string[];
+    warnings: string[];
+}
+/**
+ * Diff two runs of the same suite, so a rating-prompt edit produces a signal rather than a vibe.
+ *
+ * Three separate lists, because they answer different questions: verdict regressions are what a CI
+ * gate reads, verdict fixes are what a change claims to have done, and RECLASSIFICATIONS are what
+ * a prompt edit actually moved — a case can keep its verdict while the label underneath it changes,
+ * and that is exactly the drift a pass-rate comparison cannot see.
+ */
+export declare function diffRuns(before: EvalSuiteSummary, after: EvalSuiteSummary): RunDiff;
+/** Render a {@link RunDiff} as plain lines. */
+export declare function renderRunDiff(diff: RunDiff): string[];
+/** Does this suite declare a sweep? Small helper so the command reads declaratively. */
+export declare function suiteSweep(suite: EvalSuite): EvalSweep | undefined;