@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.26

package/dist/evalSuite.js

@@ -1,6 +1,8 @@
 import { parse as parseYaml } from 'yaml';
 import * as z from 'zod';
-import { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+import { APPROVAL_RUNGS, isApprovalRung } from '@gaunt-sloth/core/config/shell-policy.js';
+import { DEFAULT_EVAL_PASS_THRESHOLD, FORCED_BY_ASSERTIONS, FORCED_BY_MECHANISMS, } from '#src/evalTypes.js';
+import { parseMetricPredicate } from '#src/metrics.js';
 /**
  * Raw suite-file shape (snake_case, as authored). BATCH-12 adds the identity matrix on top of the
  * BATCH-10 assertion set: a suite-level `identities` list, and a per-case `expect:` array of
@@ -84,6 +86,15 @@ const RawAssertionsSchema = z.object({
     // BATCH-21 tool-RESULT assertions (gth-agent target only, enforced after normalization below).
     must_error: z.array(z.string()).optional(),
     tool_result_json_path: z.array(RawToolResultJsonPathCheckSchema).optional(),
+    // BATCH-25 CLASSIFICATION assertions. They sit in the assertion bundle — not on the case — so
+    // they inherit identity scoping AND per-turn scoping for free; a multi-round negotiation case
+    // needs a different expected action per round, which a case-level field could not express.
+    // Validated against the suite's `classification` enums after normalization.
+    expect_label: z.string().optional(),
+    expect_action: z.string().optional(),
+    // BATCH-25 Half B — assert WHICH deterministic mechanism of the approvals gate decided this
+    // round. `rater` target only; desugars to a `must_contain` on that mechanism's rationale marker.
+    forced_by: z.string().optional(),
     judge: z.string().optional(),
 });
 const RawExpectationSchema = RawAssertionsSchema.extend({
@@ -114,6 +125,63 @@ const RawCaseSchema = RawAssertionsSchema.extend({
     // each turn for a multi-turn case). Enforced in code.
     turns: z.array(RawTurnSchema).optional(),
     pass_threshold: z.number().min(0).max(10).optional(),
+    // BATCH-25: the case's family tags. Unlike the assertion keys these ARE case-level — a family is a
+    // property of the case, identical across identities and rounds — so they are legal on a multi-turn
+    // case too.
+    tags: z.array(z.string()).optional(),
+    // BATCH-25: this case must be decided with zero model calls (the hardline-floor / ambiguity
+    // families). Requires a target that can classify deterministically; rejected otherwise.
+    model_free: z.boolean().optional(),
+});
+/** BATCH-25 — how a classification value is read from an answer: the bare string `answer` (the
+ * trimmed answer, matched against the enum) or `{ json_path: "…" }` (the same minimal path resolver
+ * `json_path` assertions use). No fuzzy/substring mode exists, deliberately. */
+const RawExtractorSchema = z.union([
+    z.literal('answer'),
+    z.object({ json_path: z.string().min(1, 'json_path extractor needs a non-empty path') }),
+]);
+/** BATCH-25 — the suite's `classification:` block: the enum that gives the confusion matrix its
+ * axes, plus how to read a value out of the SUT's answer. */
+const RawClassificationSchema = z.object({
+    labels: z.array(z.string()).min(1, 'classification.labels must declare at least one label'),
+    actions: z.array(z.string()).optional(),
+    label_from: RawExtractorSchema.optional(),
+    action_from: RawExtractorSchema.optional(),
+});
+/** A predicate list, written as one string or a list of strings. Entries are ANDed. */
+const RawPredicateListSchema = z.union([z.string(), z.array(z.string())]);
+/** BATCH-25 — one declared metric. `over` (the denominator) is OPTIONAL and its absence means the
+ * WHOLE scored corpus; that default is the point, not a convenience. */
+const RawMetricSchema = z.object({
+    name: z.string().min(1, 'metric name must be a non-empty string'),
+    description: z.string().optional(),
+    where: RawPredicateListSchema,
+    over: RawPredicateListSchema.optional(),
+    // FRACTION thresholds (0..1).
+    max: z.number().optional(),
+    min: z.number().optional(),
+    // COUNT thresholds — absolute case counts, invariant to corpus size. Mutually exclusive with the
+    // fraction form on a single metric (enforced in code: two thresholds on one gate would have no
+    // defined precedence).
+    max_count: z.number().optional(),
+    min_count: z.number().optional(),
+    gate: z.enum(['fail', 'report']).optional(),
+});
+/** BATCH-25 — one sweep axis value. `model` reaches the config through BATCH-1's supported
+ * `initConfig({ model })` seam (a genuinely fresh `.llm`); `config` is a deep merge of plain data. */
+const RawSweepValueSchema = z.object({
+    name: z.string().min(1, 'sweep value name must be a non-empty string'),
+    model: z.string().optional(),
+    config: z.record(z.string(), z.unknown()).optional(),
+});
+/** BATCH-25 — the sweep: named axes whose cartesian product is the set of runs (`rung × model`). */
+const RawSweepSchema = z.object({
+    axes: z
+        .array(z.object({
+        name: z.string().min(1, 'sweep axis name must be a non-empty string'),
+        values: z.array(RawSweepValueSchema).min(1, 'a sweep axis must declare at least one value'),
+    }))
+        .min(1, 'sweep must declare at least one axis'),
 });
 const RawSuiteSchema = z.object({
     target: z.object({
@@ -124,6 +192,9 @@ const RawSuiteSchema = z.object({
         // optional debug label. Both are ignored for a `gth-agent` target.
         url: z.string().optional(),
         agent_id: z.string().optional(),
+        // BATCH-25 Half B: the approvals rung a `rater` target rates at (required for that target,
+        // validated below against the declared ladder; ignored by every other target).
+        rung: z.string().optional(),
     }),
     // BATCH-10 Task 2: optional identity profile whose model judges the cases. A top-level sibling of
     // `target`/`defaults`/`cases`, and distinct from `target.profile` (which selects the SUT and is
@@ -140,6 +211,11 @@ const RawSuiteSchema = z.object({
         pass_threshold: z.number().min(0).max(10).optional(),
     })
         .optional(),
+    // BATCH-25: the classifier layer. All three are optional and inert when absent, so every #405-era
+    // suite parses to exactly the same `EvalSuite` it did before.
+    classification: RawClassificationSchema.optional(),
+    metrics: z.array(RawMetricSchema).optional(),
+    sweep: RawSweepSchema.optional(),
     cases: z.array(RawCaseSchema).min(1, 'suite must declare at least one case'),
 });
 /** The BATCH-10 assertion keys as authored on a flat case (used to detect "declared both flat
@@ -156,6 +232,12 @@ const FLAT_ASSERTION_KEYS = [
     'json_path',
     'must_error',
     'tool_result_json_path',
+    // BATCH-25 — the classification assertions MUST be listed here. This array drives BOTH the
+    // flat-vs-`expect:` exclusivity check and the multi-turn "case-level assertions are rejected"
+    // check; omitting them would let a suite declare both surfaces and have one silently ignored.
+    'expect_label',
+    'expect_action',
+    'forced_by',
     'judge',
 ];
 /** A plain profile-name pattern — same as a case id: alphanumerics, dashes, underscores, dots. This
@@ -172,8 +254,8 @@ const IDENTITY_NAME_RE = /^[\w.-]+$/;
  * Rejects, with a clear message, at parse time (never silently no-ops or defers to run time):
  * - Malformed YAML.
  * - A suite shape that doesn't match {@link RawSuiteSchema} (missing/wrong-typed fields).
- * - `target.type` other than `"gth-agent"`, `"adk-agent"`, or `"ag-ui"` — other pluggable CLI/HTTP
- *   targets are out of scope.
+ * - `target.type` other than `"gth-agent"`, `"adk-agent"`, `"ag-ui"`, or `"rater"` — other pluggable
+ *   CLI/HTTP targets are out of scope.
  * - `target.profile` set to anything other than `"default"`/absent — a single suite-wide profile
  *   switch is the `--identities` direction, replaced by the suite-level `identities` list.
  * - A `"adk-agent"` (BATCH-14) target missing its `url`; an `adk-agent` suite that ALSO uses the
@@ -204,6 +286,16 @@ const IDENTITY_NAME_RE = /^[\w.-]+$/;
  * - A tool-RESULT assertion (`must_error` / `tool_result_json_path`, BATCH-21) against an
  *   `adk-agent` OR `ag-ui` target — tool results exist only on the in-process `gth-agent` target
  *   (the AG-UI wire streams call names but no result payloads; A2A exposes no tool trace at all).
+ * - A `"rater"` (BATCH-25 Half B) target missing its `rung`, naming one that is not on the approvals
+ *   ladder, or carrying a `profile`; a `rater` suite with no `classification:` block (the classifier
+ *   layer would be inert and every case would silently run through the ordinary agent); a `rater`
+ *   suite using the `identities` matrix (the classification seam is per-case, not per-identity); or
+ *   a `rater` suite using ANY tool assertion (no agent runs, so there is no trace to grade).
+ * - `model_free: true` on a case of any target EXCEPT `rater` — running a case through an agent IS
+ *   a model call, so the flag could never bite; and `model_free: true` on a case that ALSO declares
+ *   a `judge:` rubric, since the judge is a second model call the target's `modelCalls` cannot see.
+ * - `forced_by` on any target except `rater`, or naming something that is not a mechanism of the
+ *   approvals gate.
  * - A `judge_profile` containing a path separator or `..`.
  *
  * @param yamlText Raw suite file content.
@@ -275,11 +367,33 @@ export function parseEvalSuite(yamlText, sourcePath) {
         }
         target = { type: 'ag-ui', url, agentId };
     }
+    else if (data.target.type === 'rater') {
+        // BATCH-25 Half B: gth's own approvals rater, graded as a classifier. The `rung` is REQUIRED —
+        // the same label produces a different ACTION per rung, so a rater suite that did not say which
+        // rung it rates at would report action numbers that mean nothing. Validated against core's
+        // ladder here (not by the zod enum) so the message can list the rungs that exist.
+        const rung = data.target.rung?.trim();
+        if (!rung) {
+            throw new Error(`Invalid eval suite${suffix}: a "rater" target requires a \`rung\` — the approvals rung the ` +
+                'corpus is rated at (e.g. `target: { type: rater, rung: auto-safe }`), because the same ' +
+                `outcome maps to a different action per rung. One of: ${APPROVAL_RUNGS.join(', ')}.`);
+        }
+        if (!isApprovalRung(rung)) {
+            throw new Error(`Invalid eval suite${suffix}: unsupported target.rung "${rung}" — the approvals ladder is ` +
+                `${APPROVAL_RUNGS.join(', ')}.`);
+        }
+        if (data.target.profile !== undefined) {
+            throw new Error(`Invalid eval suite${suffix}: a "rater" target does not take a \`profile\` — the model that ` +
+                "rates is the run's own (or the one `approvals.rater` names); omit `target.profile`.");
+        }
+        target = { type: 'rater', rung };
+    }
     else {
         throw new Error(`Invalid eval suite${suffix}: unsupported target.type "${data.target.type}" — this version ` +
             'of `gth eval` supports "gth-agent" (in-process), "adk-agent" (an external Google ADK ' +
-            'agent over A2A), and "ag-ui" (an external agent over the AG-UI protocol); other pluggable ' +
-            'CLI/HTTP targets are future scope.');
+            'agent over A2A), "ag-ui" (an external agent over the AG-UI protocol), and "rater" (gth\'s ' +
+            'own approvals rater, graded as a classifier); other pluggable CLI/HTTP targets are future ' +
+            'scope.');
     }
     // Suite-level identity matrix (BATCH-12). Validate the names here (plain, path-safe, unique) so
     // later stages — output filenames, the `expect:` identity-membership check, the command's
@@ -327,6 +441,45 @@ export function parseEvalSuite(yamlText, sourcePath) {
             'target — identity profiles select per-identity gth configs, which do not apply to an ' +
             'external AG-UI agent. Remove `identities`, or use a `gth-agent` target.');
     }
+    // BATCH-25 Half B — the same honest boundary for the `rater` target. The `identities` matrix runs
+    // each case once per identity, and the classifier seam (`RunClassifyFn`) takes no identity: one
+    // classifier is built from ONE config, so a matrix would run every identity through the same
+    // rater and report N identical columns as if they were evidence. The rater axis that IS
+    // meaningful — a different rating model — is the `sweep`, which this target supports.
+    if (target.type === 'rater' && identities !== undefined) {
+        throw new Error(`Invalid eval suite${suffix}: the \`identities\` matrix is not supported for a "rater" ` +
+            'target — the classification seam is per-case, not per-identity, so every identity would ' +
+            'be rated by the same model. Use a `sweep` to vary the rating model/rung instead.');
+    }
+    // BATCH-25 — the classifier layer. Parsed BEFORE the cases, because every `expect_label` /
+    // `expect_action` and every metric literal is validated against these enums: a typo'd value is a
+    // suite error here rather than an assertion that can never pass (or, worse, a metric that reports
+    // a permanent, trusted zero).
+    const classification = buildClassificationSpec(data.classification, suffix, target.type !== 'rater');
+    const metrics = buildMetricSpecs(data.metrics, classification, suffix);
+    const sweep = buildSweep(data.sweep, suffix);
+    // BATCH-25 Half B — a `rater` suite MUST declare `classification:`. Without it the whole
+    // classifier layer is inert by design (`runEvalSuite` only consults an injected classifier for a
+    // suite that declares one), so the target would be built, ignored, and every case would fall
+    // through to the ordinary agent — sending shell commands to a chat model and grading its prose.
+    // That is the one failure mode this target must never have, and it is statically detectable.
+    if (target.type === 'rater' && classification === undefined) {
+        throw new Error(`Invalid eval suite${suffix}: a "rater" target requires a \`classification:\` block — it ` +
+            "declares the label/action enum the rater's verdicts are graded against. Without it the " +
+            'classifier layer is inert and the cases would be run through the ordinary agent instead.');
+    }
+    // BATCH-25 — a config sweep overrides the gth config the SUT is built from. An `adk-agent` /
+    // `ag-ui` target runs OUT OF PROCESS with its own model, tools and auth, so there is no config to
+    // sweep: the overrides would silently apply to the judge alone and the comparison table would
+    // report cells that differ in nothing. Reject it, the same way the `identities` matrix is
+    // rejected for those targets and for the same reason — a false-scope suite is a bug, not
+    // something to run half of.
+    if (sweep !== undefined && (target.type === 'adk-agent' || target.type === 'ag-ui')) {
+        throw new Error(`Invalid eval suite${suffix}: a \`sweep\` is not supported for a "${target.type}" target — ` +
+            'the agent runs out-of-process with its own config, so config overrides would change ' +
+            'nothing about the SUT. Sweep a `gth-agent` target, or vary the external agent yourself ' +
+            'and run the suite once per variant.');
+    }
     const suiteDefaultThreshold = data.defaults?.pass_threshold ?? DEFAULT_EVAL_PASS_THRESHOLD;
     const seenIds = new Set();
     const cases = data.cases.map((rawCase, index) => {
@@ -371,6 +524,8 @@ export function parseEvalSuite(yamlText, sourcePath) {
                         turnIndex,
                         declaredIdentities,
                         identities,
+                        classification,
+                        targetType: target.type,
                     }),
                 };
             });
@@ -392,14 +547,62 @@ export function parseEvalSuite(yamlText, sourcePath) {
                         turnIndex: undefined,
                         declaredIdentities,
                         identities,
+                        classification,
+                        targetType: target.type,
                     }),
                 },
             ];
         }
+        // BATCH-25 — family tags. De-duplicated, blanks dropped, order preserved: they are the axis of
+        // every per-tag sub-score, and a duplicate would double-count a case within its own family.
+        const seenTags = new Set();
+        const tags = [];
+        for (const rawTag of rawCase.tags ?? []) {
+            const tag = rawTag.trim();
+            if (tag.length === 0) {
+                throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) has a blank tag — ` +
+                    'a tag names a case family and is a per-tag sub-score axis, so it must be non-empty.');
+            }
+            if (seenTags.has(tag))
+                continue;
+            seenTags.add(tag);
+            tags.push(tag);
+        }
+        // BATCH-25 — `model_free` requires a target that classifies DETERMINISTICALLY and reports its
+        // own model-call count. Only the `rater` target does (Half B): a gth-agent / adk-agent / ag-ui
+        // case IS a model call by construction. Reject it elsewhere rather than accept a flag that
+        // would silently mean nothing — an assertion that cannot bite is worse than an absent one.
+        const modelFree = rawCase.model_free === true;
+        if (modelFree && target.type !== 'rater') {
+            throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) declares ` +
+                `\`model_free: true\`, which the "${target.type}" target cannot honour — running a case ` +
+                'through an agent IS a model call. `model_free` needs a classification target that ' +
+                "decides deterministically and reports its own model-call count (BATCH-25 Half B's " +
+                '`rater` target). Remove the flag, or use `target: { type: rater, rung: … }`.');
+        }
+        // BATCH-25 Half B — `model_free` must mean NO model call from ANY path, not just from the
+        // classify path. The runner enforces it against the TARGET's reported `modelCalls`, and the
+        // judge is a second, independent model call: a case declaring both `model_free: true` and a
+        // `judge:` rubric would bill an LLM call and still report `modelCalls: 0` and PASS. A contract
+        // that exists to make a claim checkable must not have a hole in it.
+        if (modelFree) {
+            for (const turn of turns) {
+                for (const expectation of turn.expectations) {
+                    if (expectation.judgeRubric !== undefined) {
+                        throw new Error(`Invalid eval suite${suffix}: case "${rawCase.id}" (index ${index}) declares ` +
+                            '`model_free: true` AND a `judge:` rubric — the judge is a model call, so the case ' +
+                            "would not be free and its reported model-call count (the target's, which the " +
+                            'judge is not part of) would say it was. Drop the rubric, or drop `model_free`.');
+                    }
+                }
+            }
+        }
         return {
             id: rawCase.id,
             turns,
             passThreshold: rawCase.pass_threshold ?? suiteDefaultThreshold,
+            tags,
+            modelFree,
         };
     });
     // BATCH-14 (design point 4) — the HONEST tool-call boundary for the ADK target. A2A's wire content
@@ -445,6 +648,30 @@ export function parseEvalSuite(yamlText, sourcePath) {
             }
         }
     }
+    // BATCH-25 Half B — the honest boundary for the `rater` target: it runs NO agent and calls NO
+    // tool, so there is no tool trace and there are no tool results. `must_not_call` would pass
+    // vacuously against the empty trace (the silent false green an eval must never produce) and
+    // `must_call` would fail for a reason that has nothing to do with the rater. Reject all four,
+    // naming the case, the same way the external targets reject what their wire cannot carry.
+    if (target.type === 'rater') {
+        for (const evalCase of cases) {
+            for (const turn of evalCase.turns) {
+                for (const expectation of turn.expectations) {
+                    if (expectation.mustCall.length > 0 ||
+                        expectation.mustNotCall.length > 0 ||
+                        expectation.mustError.length > 0 ||
+                        expectation.toolResultJsonPath.length > 0) {
+                        throw new Error(`Invalid eval suite${suffix}: case "${evalCase.id}" uses a tool assertion ` +
+                            '(`must_call` / `must_not_call` / `must_error` / `tool_result_json_path`), which a ' +
+                            '"rater" target cannot carry — it rates a command string and never runs an agent, ' +
+                            'so there is no tool trace to grade (and a vacuous pass is worse than no ' +
+                            'assertion). Grade the rating with `expect_label` / `expect_action`, or the ' +
+                            "rater's own explanation with the content assertions.");
+                    }
+                }
+            }
+        }
+    }
     // Normalize a blank/whitespace-only judge_profile to undefined (= no separate judge) so the CLI's
     // resolution treats it the same as absent.
     const judgeProfile = data.judge_profile?.trim() || undefined;
@@ -459,9 +686,220 @@ export function parseEvalSuite(yamlText, sourcePath) {
         target,
         judgeProfile,
         identities,
+        classification,
+        metrics,
+        sweep,
         cases,
     };
 }
+/**
+ * BATCH-25 — normalize the suite's `classification:` block.
+ *
+ * `label_from` defaults to `answer` (the trimmed answer matched case-insensitively against the
+ * declared enum), which suits a suite whose prompt says "reply with exactly one of: …". An
+ * `action_from` has NO default: without it the suite has no action dimension, and `expect_action` is
+ * then a parse error rather than an assertion that could never be graded.
+ *
+ * BATCH-25 Half B — `extractsFromAnswer` is false for a target that CLASSIFIES (the `rater`), which
+ * reports its label and action directly instead of leaving them to be read out of an answer. The
+ * extractors are then irrelevant, so requiring an `action_from` alongside `actions:` would force
+ * every rater suite to write a line that is never consulted — and a config line that does nothing
+ * is one a reader will eventually believe.
+ */
+function buildClassificationSpec(raw, suffix, extractsFromAnswer) {
+    if (raw === undefined)
+        return undefined;
+    const labels = normalizeEnum(raw.labels, 'classification.labels', suffix);
+    const actions = normalizeEnum(raw.actions ?? [], 'classification.actions', suffix);
+    const actionFrom = raw.action_from ? normalizeExtractor(raw.action_from) : undefined;
+    if (actions.length === 0 && actionFrom !== undefined) {
+        throw new Error(`Invalid eval suite${suffix}: \`classification.action_from\` is declared but ` +
+            '`classification.actions` is empty — an extractor with no enum to match against can only ' +
+            'ever produce "(unrecognized)".');
+    }
+    if (actions.length > 0 && actionFrom === undefined && extractsFromAnswer) {
+        throw new Error(`Invalid eval suite${suffix}: \`classification.actions\` is declared but ` +
+            '`classification.action_from` is not — declare how an action is read out of the answer ' +
+            '(e.g. `action_from: { json_path: "$.action" }`), or drop `actions`.');
+    }
+    return {
+        labels,
+        actions,
+        labelFrom: raw.label_from ? normalizeExtractor(raw.label_from) : { kind: 'answer' },
+        actionFrom,
+    };
+}
+/** Validate one declared enum: non-blank, unique, and free of the parenthesized synthetic buckets
+ * (`(unrecognized)` / `(none)`), which the matrix reserves for "no declared value matched" and "no
+ * expectation". A declared value colliding with one of those would make the two indistinguishable. */
+function normalizeEnum(values, field, suffix) {
+    const seen = new Set();
+    const out = [];
+    for (const rawValue of values) {
+        const value = rawValue.trim();
+        if (value.length === 0) {
+            throw new Error(`Invalid eval suite${suffix}: \`${field}\` contains a blank value.`);
+        }
+        if (!/^[\w.-]+$/.test(value)) {
+            throw new Error(`Invalid eval suite${suffix}: \`${field}\` value "${value}" must be a plain token ` +
+                '(alphanumeric, dashes, underscores, dots) — enum values are matrix axis labels and ' +
+                'metric-predicate literals.');
+        }
+        if (seen.has(value)) {
+            throw new Error(`Invalid eval suite${suffix}: duplicate \`${field}\` value "${value}".`);
+        }
+        seen.add(value);
+        out.push(value);
+    }
+    return out;
+}
+/** `"answer"` | `{ json_path }` → the normalized extractor. */
+function normalizeExtractor(raw) {
+    return raw === 'answer' ? { kind: 'answer' } : { kind: 'json_path', path: raw.json_path };
+}
+/** Accept a predicate list written as one string or a list of strings; entries are ANDed. */
+function toPredicateList(raw) {
+    if (raw === undefined)
+        return [];
+    return Array.isArray(raw) ? raw : [raw];
+}
+/**
+ * BATCH-25 — normalize the suite's `metrics:` list.
+ *
+ * Two rules here are load-bearing rather than defensive:
+ * - a metric requires a `classification:` block (there is nothing to read otherwise);
+ * - every predicate literal is checked against the declared enums by
+ *   {@link ../metrics.js parseMetricPredicate}, because a typo'd value produces an unsatisfiable
+ *   predicate — a metric that reports a permanent, and trusted, zero.
+ *
+ * `gate` defaults to `fail` when a threshold is declared: a threshold nobody acts on is a comment.
+ * An author who wants a reported-but-not-gating number writes `gate: report` (the corpus plan's
+ * `over_escalation`).
+ */
+function buildMetricSpecs(raw, classification, suffix) {
+    if (raw === undefined || raw.length === 0)
+        return [];
+    if (!classification) {
+        throw new Error(`Invalid eval suite${suffix}: \`metrics\` requires a \`classification:\` block — a metric ` +
+            'reads labels/actions, so the suite must declare which ones exist.');
+    }
+    const seen = new Set();
+    return raw.map((rawMetric, index) => {
+        const name = rawMetric.name.trim();
+        if (seen.has(name)) {
+            throw new Error(`Invalid eval suite${suffix}: duplicate metric name "${name}".`);
+        }
+        seen.add(name);
+        const where = toPredicateList(rawMetric.where);
+        if (where.length === 0) {
+            throw new Error(`Invalid eval suite${suffix}: metric "${name}" (index ${index}) has an empty \`where\` — ` +
+                'a metric with no numerator predicate counts every case in its denominator, which is a ' +
+                'coverage report, not a metric.');
+        }
+        const rawOver = toPredicateList(rawMetric.over);
+        const at = `Invalid eval suite${suffix}: metric "${name}" (index ${index})`;
+        const wherePredicates = where.map((predicate) => parseMetricPredicate(predicate, classification, `${at} \`where\``));
+        const overPredicates = rawMetric.over === undefined
+            ? undefined
+            : rawOver.map((predicate) => parseMetricPredicate(predicate, classification, `${at} \`over\``));
+        if (overPredicates !== undefined && overPredicates.length === 0) {
+            throw new Error(`${at} has an empty \`over\` — omit the key entirely to score the WHOLE corpus (the ` +
+                'default, and the one that cannot go blind).');
+        }
+        // A metric declares its thresholds in ONE unit. Mixing them would put two thresholds on one
+        // gate with no defined precedence, and would leave every downstream reader — the console, the
+        // comparison table, results.json — guessing whether `2` meant two cases or 200%.
+        const hasFraction = rawMetric.max !== undefined || rawMetric.min !== undefined;
+        const hasCount = rawMetric.max_count !== undefined || rawMetric.min_count !== undefined;
+        if (hasFraction && hasCount) {
+            throw new Error(`${at} declares BOTH fraction thresholds (\`max\`/\`min\`) and count thresholds ` +
+                '(`max_count`/`min_count`) — a metric gates in one unit. Use counts when the target is ' +
+                '"at most N cases" (invariant to corpus size), fractions when it is "at most X% of the ' +
+                'denominator".');
+        }
+        for (const [field, value] of [
+            ['max', rawMetric.max],
+            ['min', rawMetric.min],
+        ]) {
+            if (value !== undefined && (value < 0 || value > 1)) {
+                throw new Error(`${at} has \`${field}: ${value}\` — \`${field}\` is a FRACTION of the denominator ` +
+                    `(0..1), so a zero-tolerance gate is \`max: 0\` and a full-recall gate is \`min: 1\`. ` +
+                    `For an absolute target like "at most ${value} case(s)", use \`${field}_count: ` +
+                    `${value}\` — it does not drift as the corpus grows.`);
+            }
+        }
+        for (const [field, value] of [
+            ['max_count', rawMetric.max_count],
+            ['min_count', rawMetric.min_count],
+        ]) {
+            if (value !== undefined && (!Number.isInteger(value) || value < 0)) {
+                throw new Error(`${at} has \`${field}: ${value}\` — a count threshold is a whole, non-negative number ` +
+                    'of cases. For a fractional target use `max`/`min` (0..1).');
+            }
+        }
+        return {
+            name,
+            description: rawMetric.description?.trim() || undefined,
+            where: wherePredicates,
+            over: overPredicates,
+            max: rawMetric.max,
+            min: rawMetric.min,
+            maxCount: rawMetric.max_count,
+            minCount: rawMetric.min_count,
+            gate: rawMetric.gate ?? 'fail',
+        };
+    });
+}
+/** BATCH-25 — normalize the `sweep:` block. Axis and value names double as output-dir components,
+ * so they get the same path-safe validation case ids and identity names get. */
+function buildSweep(raw, suffix) {
+    if (raw === undefined)
+        return undefined;
+    const seenAxes = new Set();
+    const axes = raw.axes.map((rawAxis) => {
+        const axisName = rawAxis.name.trim();
+        assertPathSafeToken(axisName, `sweep axis name`, suffix);
+        if (seenAxes.has(axisName)) {
+            throw new Error(`Invalid eval suite${suffix}: duplicate sweep axis "${axisName}".`);
+        }
+        seenAxes.add(axisName);
+        const seenValues = new Set();
+        const values = rawAxis.values.map((rawValue) => {
+            const valueName = rawValue.name.trim();
+            assertPathSafeToken(valueName, `sweep value name (axis "${axisName}")`, suffix);
+            if (seenValues.has(valueName)) {
+                throw new Error(`Invalid eval suite${suffix}: duplicate sweep value "${valueName}" on axis "${axisName}".`);
+            }
+            seenValues.add(valueName);
+            if (rawValue.config && 'llm' in rawValue.config) {
+                throw new Error(`Invalid eval suite${suffix}: sweep value "${axisName}=${valueName}" sets \`config.llm\`. ` +
+                    '`llm` holds a CONSTRUCTED model instance, not data — merging into it produces a ' +
+                    'half-built object. Use the sibling `model:` key, which rebuilds the model through the ' +
+                    'provider (a genuinely fresh instance).');
+            }
+            if (rawValue.model === undefined && rawValue.config === undefined) {
+                throw new Error(`Invalid eval suite${suffix}: sweep value "${axisName}=${valueName}" declares neither ` +
+                    '`model:` nor `config:` — a cell that overrides nothing is an unnamed duplicate run.');
+            }
+            return {
+                name: valueName,
+                model: rawValue.model?.trim() || undefined,
+                config: rawValue.config,
+            };
+        });
+        return { name: axisName, values };
+    });
+    return { axes };
+}
+/** A plain, path-safe token — the same rule case ids and identity names follow, for the same reason
+ * (these names become output-directory components). */
+function assertPathSafeToken(value, what, suffix) {
+    if (!/^[\w.-]+$/.test(value) || value.includes('..')) {
+        throw new Error(`Invalid eval suite${suffix}: ${what} "${value}" must be a plain token (alphanumeric, ` +
+            'dashes, underscores, dots) — it becomes an output-directory component, so path ' +
+            'separators and ".." are rejected.');
+    }
+}
 /**
  * Normalize ONE turn's raw assertion surface — a single-`prompt` case's case-level fields, or one
  * `turns:` entry — into its {@link EvalExpectation} blocks. Shared by the single-turn and multi-turn
@@ -495,6 +933,8 @@ function buildTurnExpectations(raw, ctx) {
             turnIndex: ctx.turnIndex,
             blockIndex,
             declaredIdentities: ctx.declaredIdentities,
+            classification: ctx.classification,
+            targetType: ctx.targetType,
         }));
     }
     else {
@@ -507,6 +947,8 @@ function buildTurnExpectations(raw, ctx) {
                 turnIndex: ctx.turnIndex,
                 blockIndex: undefined,
                 declaredIdentities: ctx.declaredIdentities,
+                classification: ctx.classification,
+                targetType: ctx.targetType,
             }),
         ];
     }
@@ -526,6 +968,42 @@ function buildTurnExpectations(raw, ctx) {
     }
     return expectations;
 }
+/**
+ * BATCH-25 Half B — desugar `forced_by: <mechanism>` into the content assertion that grades it.
+ *
+ * ## Why a case needs this at all
+ *
+ * A `model_free` case cannot be graded on its `action`. Without a verdict, core's decision mapping
+ * substitutes its fail-closed one and returns the SAME action for every command at a rated rung, so
+ * `expect_action: escalate` passes for `ls -la` exactly as it does for `rm -rf $(echo /)` — and
+ * would still pass with the hardline floor and both preflights deleted. A model-free assertion is
+ * only a regression gate if a DIFFERENT command would get it wrong, and the one signal that
+ * satisfies that is WHICH deterministic mechanism decided the command. The `rater` target reports
+ * it in the rationale; this turns the corpus's own `forced_by` / `floor_refuses` vocabulary into the
+ * `must_contain` that reads it, so transcribing a corpus case stays a copy.
+ *
+ * `rater`-only: no other target has a deterministic layer to attribute a decision to, and an
+ * assertion that can never be satisfied is as bad as one that always is.
+ */
+function parseForcedBy(raw, ctx) {
+    const mechanism = raw?.trim();
+    if (!mechanism)
+        return undefined;
+    const turnPart = ctx.turnIndex === undefined ? '' : ` turn ${ctx.turnIndex}`;
+    const where = ctx.blockIndex === undefined
+        ? `case "${ctx.caseId}" (index ${ctx.caseIndex})${turnPart}`
+        : `case "${ctx.caseId}" (index ${ctx.caseIndex})${turnPart} expect block ${ctx.blockIndex}`;
+    if (ctx.targetType !== 'rater') {
+        throw new Error(`Invalid eval suite${ctx.suffix}: ${where} uses \`forced_by\`, which only the "rater" target ` +
+            `can grade — it names a deterministic mechanism of the approvals gate (${FORCED_BY_MECHANISMS.join(', ')}), and a "${ctx.targetType}" target has none to report. Remove it, or use ` +
+            '`target: { type: rater, rung: … }`.');
+    }
+    if (!FORCED_BY_MECHANISMS.includes(mechanism)) {
+        throw new Error(`Invalid eval suite${ctx.suffix}: ${where} declares \`forced_by: ${mechanism}\`, which is not ` +
+            `a mechanism of the approvals gate. One of: ${FORCED_BY_MECHANISMS.join(', ')}.`);
+    }
+    return mechanism;
+}
 /**
  * Normalize one raw assertion bundle (a flat case's case-level fields, or one `expect:` block) into
  * an {@link EvalExpectation}: default arrays to `[]`, compile regexes at parse time, validate
@@ -554,7 +1032,16 @@ function buildExpectation(raw, blockIdentities, ctx) {
         }
         identities = blockIdentities;
     }
-    const mustContain = raw.must_contain ?? [];
+    // `forced_by` is BOTH graded and load-bearing for how the round is driven, so the mechanism is
+    // kept as well as desugared: the marker below is what the grader reads, and `forcedBy` on the
+    // returned expectation is what lets the target hand the gate a rating to override (see
+    // `mechanismNeedsPermissiveRating`). Deriving one from the other later would mean matching marker
+    // TEXT back to a mechanism.
+    const forcedBy = parseForcedBy(raw.forced_by, ctx);
+    const mustContain = [
+        ...(raw.must_contain ?? []),
+        ...(forcedBy === undefined ? [] : [FORCED_BY_ASSERTIONS[forcedBy]]),
+    ];
     const mustNotContain = raw.must_not_contain ?? [];
     const shouldContainAny = raw.should_contain_any ?? [];
     const mustCall = raw.must_call ?? [];
@@ -605,6 +1092,37 @@ function buildExpectation(raw, blockIdentities, ctx) {
             return { tool: entry.tool, path: entry.path, equals: entry.equals };
         return { tool: entry.tool, path: entry.path };
     });
+    // BATCH-25 classification assertions. Validated against the suite's declared enums HERE (not
+    // later) so a typo'd label is a suite error rather than a case that can never pass.
+    const expectLabel = raw.expect_label?.trim() || undefined;
+    const expectAction = raw.expect_action?.trim() || undefined;
+    if (expectLabel !== undefined) {
+        if (!ctx.classification) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} uses \`expect_label\` but the suite declares no ` +
+                '`classification:` block — add one naming the label enum (e.g. `classification: { ' +
+                'labels: [safe, destructive] }`), which is also what gives the confusion matrix its axes.');
+        }
+        if (!ctx.classification.labels.includes(expectLabel)) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} expects label "${expectLabel}", which is not in ` +
+                `the suite's \`classification.labels\` (${ctx.classification.labels.join(', ')}).`);
+        }
+    }
+    if (expectAction !== undefined) {
+        if (!ctx.classification) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} uses \`expect_action\` but the suite declares ` +
+                'no `classification:` block — add one declaring `actions:` and `action_from:`.');
+        }
+        if (ctx.classification.actions.length === 0) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} uses \`expect_action\` but the suite's ` +
+                '`classification:` block declares no `actions:` enum — an action assertion with no ' +
+                'action dimension could never be graded, and an ungradeable assertion that silently ' +
+                'passes is the worst outcome for an eval tool.');
+        }
+        if (!ctx.classification.actions.includes(expectAction)) {
+            throw new Error(`Invalid eval suite${ctx.suffix}: ${where} expects action "${expectAction}", which is not ` +
+                `in the suite's \`classification.actions\` (${ctx.classification.actions.join(', ')}).`);
+        }
+    }
     const hasChecks = mustContain.length > 0 ||
         mustNotContain.length > 0 ||
         shouldContainAny.length > 0 ||
@@ -614,14 +1132,19 @@ function buildExpectation(raw, blockIdentities, ctx) {
         mustNotMatch.length > 0 ||
         jsonPath.length > 0 ||
         mustError.length > 0 ||
-        toolResultJsonPath.length > 0;
+        toolResultJsonPath.length > 0 ||
+        // BATCH-25 — a classification case whose ONLY assertion is `expect_label`/`expect_action` is the
+        // primary shape of a classifier suite; without these two clauses it would be rejected here as
+        // "no checks and no judge rubric".
+        expectLabel !== undefined ||
+        expectAction !== undefined;
     const judgeRubric = raw.judge?.trim();
     const hasJudge = !!judgeRubric;
     if (!hasChecks && !hasJudge) {
         throw new Error(`Invalid eval suite${ctx.suffix}: ${where} has no checks and no judge rubric — it must ` +
             'declare at least one of must_contain / must_not_contain / should_contain_any / must_call ' +
             '/ must_not_call / must_match / must_not_match / json_path / must_error / ' +
-            'tool_result_json_path, or a judge rubric.');
+            'tool_result_json_path / expect_label / expect_action / forced_by, or a judge rubric.');
     }
     return {
         identities,
@@ -635,7 +1158,10 @@ function buildExpectation(raw, blockIdentities, ctx) {
         jsonPath,
         mustError,
         toolResultJsonPath,
+        expectLabel,
+        expectAction,
         judgeRubric: hasJudge ? judgeRubric : undefined,
+        forcedBy,
     };
 }
 //# sourceMappingURL=evalSuite.js.map