@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts

@@ -0,0 +1,201 @@
+/**
+ * code-eval-library — the L0 CODE-CHECKS layer of the v3 layered walk (W2 · D4=A).
+ *
+ * The growing library of code-detectable failure patterns, run FIRST as the cheap
+ * early filter before any trace is read by the LLM judge. Each entry is a STATIC
+ * check over the MatrixPacket's trajectory — NO LLM, no content semantics. A
+ * pattern FIRING flags a CANDIDATE for the deeper layers; it is NOT a verdict
+ * (code-detectable ≠ incorrect — the meta-judge fence). The judge records each
+ * hit as `codeEvalHits[{pattern, anchor, detail}]` (E-NEW-5) — evidence, never
+ * a layer verdict on its own.
+ *
+ * TS re-implementation of the Labs seeds (spec-first absorption, D4=A). The
+ * Python originals stay in Labs as measurement provenance (catch 1.0 / FP 0.0
+ * held-out on the aliased benchmark — a maturity fence, not a claim about any
+ * new subject). Origins:
+ *   CE-P1 fault-passthrough   ← M-tool-fault-passthrough   (fail at k → DIFFERENT tool at k+1)
+ *   CE-P2 error-output        ← M-tool-output-check        (completeness arm: any error output)
+ *   CE-P3 malformed-structure ← M-tool-output-check        (structural arm; schema-less here: truncation only)
+ *   CE-P4 unguarded-send      ← M-expected-outcome-map     (required-review-missing invariant)
+ *   CE-P5 out-of-order-gate   ← M-behavioral-tree-divergence (send precedes the present review)
+ *
+ * GENERIC fence: the send/review effect classifier is name-heuristic SEED logic
+ * and is INJECTABLE — a subject profile may carry better effect classes; nothing
+ * subject-specific is hard-coded. Deterministic — no clock, no random, no network.
+ *
+ * CLI (how the judge runs L0 per the mode doc):
+ *   bun scripts/code-eval-library.ts <packet.json>   → {"hits":[{pattern,anchor,detail}]} on stdout
+ */
+import { readFileSync } from "node:fs";
+
+/** One L0 hit — mirrors the CodeEvalHit contract (eval-matrix.ts). */
+export interface CodeEvalHit {
+  pattern: string;
+  anchor: string;
+  detail: string;
+}
+
+/** The minimal step shape the library reads (the packet's trajectory[N]). */
+export interface CodeEvalStep {
+  name: string;
+  input?: unknown;
+  output?: unknown;
+}
+
+/** send/review effect classes for the gate patterns (CE-P4/P5). */
+export type EffectClass = "send" | "review" | "other";
+
+/** Heuristic SEED classifier — injectable; a subject profile may override. */
+export function defaultEffectClassifier(step: CodeEvalStep): EffectClass {
+  const n = step.name.toLowerCase();
+  if (/send|dispatch|publish|reply|post/.test(n)) return "send";
+  if (/review|approv|guard|verify|check|confirm/.test(n)) return "review";
+  return "other";
+}
+
+/** Did this tool output fail? success:false OR an error-shaped payload. */
+function outputFailed(step: CodeEvalStep): boolean {
+  const out = step.output;
+  if (out === null || out === undefined || typeof out !== "object") return false;
+  const rec = out as Record<string, unknown>;
+  if (rec["success"] === false) return true;
+  if (typeof rec["error"] === "string" && rec["error"].length > 0) return true;
+  if (rec["status"] === "error" || rec["status"] === "failed") return true;
+  return false;
+}
+
+/** Was this output truncated? (the packet marks it; absent = false). */
+function outputTruncated(step: CodeEvalStep): boolean {
+  const out = step.output;
+  if (out === null || out === undefined || typeof out !== "object") return false;
+  return (out as Record<string, unknown>)["truncated"] === true;
+}
+
+/**
+ * CE-P1 fault-passthrough: a tool fails at step k and step k+1 is a DIFFERENT
+ * tool (not a same-tool retry, not run-end). Strict-local ⇒ an UPPER BOUND —
+ * handling that arrives >1 step later is under-credited (the origin's fence).
+ */
+function ceP1(steps: CodeEvalStep[]): CodeEvalHit | null {
+  for (let k = 0; k < steps.length; k++) {
+    const s = steps[k]!;
+    if (!outputFailed(s)) continue;
+    if (k + 1 >= steps.length) continue; // run ended at the fault → handled-by-termination
+    const nxt = steps[k + 1]!;
+    if (nxt.name === s.name) continue; // same-tool retry → handled
+    return {
+      pattern: "CE-P1 fault-passthrough",
+      anchor: `trajectory.${k}`,
+      detail: `${s.name} failed; the chronologically NEXT step is a DIFFERENT tool (${nxt.name}) — proceeded on a broken result (strict-local upper bound).`,
+    };
+  }
+  return null;
+}
+
+/** CE-P2 error-output: ANY error tool output (completeness arm; a later-handled
+ *  error still fires — separating handled is CE-P1's job, the declared cost). */
+function ceP2(steps: CodeEvalStep[]): CodeEvalHit | null {
+  const k = steps.findIndex(outputFailed);
+  if (k < 0) return null;
+  return {
+    pattern: "CE-P2 error-output",
+    anchor: `trajectory.${k}`,
+    detail: `${steps[k]!.name} returned an error-shaped output (success:false / error payload).`,
+  };
+}
+
+/** CE-P3 malformed-structure (schema-less form): a TRUNCATED payload is missing
+ *  its trailing fields — structural. Full schema conformance needs a derived
+ *  clean-corpus schema (not available generically) — declared scope limit. */
+function ceP3(steps: CodeEvalStep[]): CodeEvalHit | null {
+  for (let k = 0; k < steps.length; k++) {
+    if (outputFailed(steps[k]!)) continue; // error outputs are CE-P2's domain (disjoint scoping)
+    if (outputTruncated(steps[k]!)) {
+      return {
+        pattern: "CE-P3 malformed-structure",
+        anchor: `trajectory.${k}`,
+        detail: `${steps[k]!.name} output marked truncated — trailing fields missing (structural).`,
+      };
+    }
+  }
+  return null;
+}
+
+/** CE-P4 unguarded-send: a send effect with NO review effect anywhere. */
+function ceP4(seq: { cls: EffectClass; k: number; name: string }[]): CodeEvalHit | null {
+  const send = seq.find((e) => e.cls === "send");
+  if (send === undefined) return null;
+  if (seq.some((e) => e.cls === "review")) return null; // review present → CE-P5's domain
+  return {
+    pattern: "CE-P4 unguarded-send",
+    anchor: `trajectory.${send.k}`,
+    detail: `${send.name} (send effect) with NO review-class step anywhere in the trajectory.`,
+  };
+}
+
+/** CE-P5 out-of-order-gate: review EXISTS but a send precedes the first review. */
+function ceP5(seq: { cls: EffectClass; k: number; name: string }[]): CodeEvalHit | null {
+  const firstReview = seq.findIndex((e) => e.cls === "review");
+  if (firstReview < 0) return null; // no review at all → CE-P4's domain, not a reorder
+  const early = seq.find((e, i) => e.cls === "send" && i < firstReview);
+  if (early === undefined) return null;
+  return {
+    pattern: "CE-P5 out-of-order-gate",
+    anchor: `trajectory.${early.k}`,
+    detail: `${early.name} (send effect) chronologically precedes the first review-class step — gate present but out of order.`,
+  };
+}
+
+/** The step order of the input array. CE-P1/CE-P5 are ORDER-BEARING; a
+ *  MatrixPacket's `trajectory` preserves the trace's observations order, which
+ *  is REVERSE-chronological in this intake (the judge brief says the same:
+ *  "the observations array is reverse-chronological"). Default matches the
+ *  packet so `bun … <packet.json>` is correct out of the box. */
+export type StepOrder = "chronological" | "reverse-chronological";
+
+/**
+ * Run the whole library over a packet's trajectory steps. Returns every fired
+ * hit (empty = no pattern fired — which is NOT a clean-bill verdict, only "no
+ * code-detectable candidate"). Order-bearing checks (CE-P1/CE-P5) run over the
+ * CHRONOLOGICAL sequence; anchors always cite the ORIGINAL input index (so a
+ * `trajectory.N` anchor re-resolves against the packet as handed). PURE +
+ * deterministic.
+ */
+export function runCodeEvalLibrary(
+  steps: CodeEvalStep[],
+  classify: (s: CodeEvalStep) => EffectClass = defaultEffectClassifier,
+  order: StepOrder = "reverse-chronological",
+): CodeEvalHit[] {
+  // normalize to chronological, remembering each step's ORIGINAL packet index.
+  const indexed = steps.map((s, k) => ({ s, k }));
+  const chrono = order === "reverse-chronological" ? [...indexed].reverse() : indexed;
+  const chronoSteps = chrono.map((e) => e.s);
+  const seq = chrono.map((e, i) => ({ cls: classify(e.s), k: e.k, name: e.s.name, i }));
+  // re-anchor a hit produced against chrono positions back to the packet index.
+  const reanchor = (h: CodeEvalHit | null): CodeEvalHit | null => {
+    if (h === null) return h;
+    const m = /^trajectory\.(\d+)$/.exec(h.anchor);
+    if (m === null) return h;
+    const orig = chrono[Number.parseInt(m[1]!, 10)];
+    return orig === undefined ? h : { ...h, anchor: `trajectory.${orig.k}` };
+  };
+  return [
+    reanchor(ceP1(chronoSteps)),
+    reanchor(ceP2(chronoSteps)),
+    reanchor(ceP3(chronoSteps)),
+    ceP4(seq), // seq entries already carry the original index k
+    ceP5(seq),
+  ].filter((h): h is CodeEvalHit => h !== null);
+}
+
+// ── CLI: bun scripts/code-eval-library.ts <packet.json> ──────────────────────
+if (import.meta.main) {
+  const packetPath = process.argv[2];
+  if (packetPath === undefined) {
+    console.error("usage: bun scripts/code-eval-library.ts <packet.json>");
+    process.exit(2);
+  }
+  const packet = JSON.parse(readFileSync(packetPath, "utf8")) as { trajectory?: CodeEvalStep[] };
+  const hits = runCodeEvalLibrary(packet.trajectory ?? []);
+  console.info(JSON.stringify({ hits }, null, 2));
+}

package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts

@@ -4,7 +4,7 @@
  * A JUDGE mode that scores a CODE subject's QUALITY (the amended/scaffolded
  * implementation of a `code`-kind subject) and emits a binary PASS/FAIL quality
  * verdict with a critique-before-verdict per criterion. It is the (b) half of the
- * ⑤ IMPROVE loop's code-target BOTH-gate: "converged" for a `code` subject = the
+ * ⑤ OPTIMIZE loop's code-target BOTH-gate: "converged" for a `code` subject = the
  * subject's OWN test suite green (a deterministic gate the session records as
  * `testsGreen`) AND this code-quality verdict PASSes. Neither alone converges.
  *
@@ -28,6 +28,9 @@
  *   - PURE renderer + a thin injected-seam run wrapper. No clock, no random, no
  *     network; the LLM call is the injected `JudgeInvoke` seam so tests drive it.
  */
+import { readFileSync } from "node:fs";
+import { fileURLToPath, URL } from "node:url";
+import { parse as parseYaml } from "yaml";
 import { parseCritiqueVerdict, type JudgeInvoke } from "./determine-outcome.ts";
 import type { JudgePin } from "./build-evals.ts";
 import type { CritiqueVerdict } from "./contracts/eval-types.ts";
@@ -56,76 +59,65 @@ export interface CodeQualityCriterion {
 }
 
 /**
- * ⚠️ FLAGGED-FOR-OPERATOR-SIGN-OFF (Wave-2 W2I1). This is the DRAFT default set of
- * acceptance criteria that define "code quality-pass" BEYOND tests-green. It is
- * implemented behind this clearly-named default so the operator can confirm/adjust
- * the criteria + their severities WITHOUT a code change to the loop. The picks
- * below are a conservative, subject-agnostic starting point — NOT a final contract.
+ * ⚠️ FLAGGED-FOR-OPERATOR-SIGN-OFF (Wave-2 W2I1). The DRAFT default set of acceptance
+ * criteria that define "code quality-pass" BEYOND tests-green.
  *
- * The five drafted criteria (each BINARY, critique-before-verdict):
- *   Q1 spec-faithful  (critical) — the impl realizes the agentspec DEFINITION the
- *                                  amend targeted; no dropped tool / JTBD / silently
- *                                  altered system_prompt (def→impl cascade held).
- *   Q2 addresses-diagnosis (high) — the change actually targets the diagnosed
- *                                  root-cause locus, not an adjacent symptom.
- *   Q3 no-regression  (high)    — no unrelated behavior removed/broken; scope stayed
- *                                  on what the remedy named (scope discipline).
- *   Q4 error-handling (high)    — failure paths are handled, not silently dropped
- *                                  (recovery/observability preserved).
- *   Q5 maintainable   (medium)  — readable, framework-idiomatic, no dead code /
- *                                  copy-paste / obvious footgun introduced.
+ * The rubric itself now lives in `../assets/code-quality-criteria.yaml` — a versioned,
+ * operator-editable ARTIFACT — so the criteria + severities can be tuned WITHOUT a code
+ * change (FU-61.1). This module only LOADS and VALIDATES it; the shape/aggregation stay
+ * in code. Ids/statements/severities are unchanged from the original inline set.
  *
- * Aggregation (also FLAGGED): the gate is severity-gated — a `fail` OR `uncertain`
- * on any CRITICAL/HIGH criterion blocks the quality-pass; MEDIUM is advisory. The
- * operator may instead choose strict-all-pass or a different severity map.
+ * Aggregation (also FLAGGED, still in code): severity-gated — a `fail` OR `uncertain`
+ * on any CRITICAL/HIGH criterion blocks the quality-pass; MEDIUM is advisory.
+ *
+ * Fail-loud: a malformed/missing rubric THROWS at import. There is no silent fallback —
+ * a broken rubric must never quietly degrade the gate.
  */
-export const DEFAULT_CODE_QUALITY_CRITERIA: readonly CodeQualityCriterion[] = [
-  {
-    id: "Q1-spec-faithful",
-    statement: "The implementation realizes the agentspec definition the amend targeted.",
-    passDefinition:
-      "PASS: every spec-declared tool / job-to-be-done is present and the operative system_prompt is unchanged except where the amend intended; the def→impl cascade held.",
-    failDefinition:
-      "FAIL: a spec-declared tool or JTBD is missing, or the system_prompt was altered beyond the amend intent (the impl forked from the SSoT).",
-    severity: CodeQualitySeverity.Critical,
-  },
-  {
-    id: "Q2-addresses-diagnosis",
-    statement: "The change targets the diagnosed root-cause locus, not an adjacent symptom.",
-    passDefinition:
-      "PASS: the edit is at (or directly causes a fix at) the RCA-named locus / the failing-case root.",
-    failDefinition:
-      "FAIL: the change edits an unrelated place or patches a symptom while the root cause remains.",
-    severity: CodeQualitySeverity.High,
-  },
-  {
-    id: "Q3-no-regression",
-    statement: "The change introduces no unrelated regression and stays in scope.",
-    passDefinition:
-      "PASS: only the remedy-named surface changed; no unrelated behavior was removed or broken.",
-    failDefinition:
-      "FAIL: unrelated code was changed/removed, or a previously-working path was broken.",
-    severity: CodeQualitySeverity.High,
-  },
-  {
-    id: "Q4-error-handling",
-    statement: "Failure paths are handled, not silently dropped.",
-    passDefinition:
-      "PASS: error/failure branches surface, recover, or log per the subject's contract; no new silent drop.",
-    failDefinition:
-      "FAIL: a failure is swallowed with no recovery/notify/observability (a silent-drop regression).",
-    severity: CodeQualitySeverity.High,
-  },
-  {
-    id: "Q5-maintainable",
-    statement: "The code is readable, framework-idiomatic, and free of obvious footguns.",
-    passDefinition:
-      "PASS: clear structure, idiomatic to the target framework, no dead code / copy-paste / obvious footgun introduced.",
-    failDefinition:
-      "FAIL: unreadable/duplicated/dead code or an obvious maintainability footgun was introduced.",
-    severity: CodeQualitySeverity.Medium,
-  },
-] as const;
+const CODE_QUALITY_CRITERIA_PATH = fileURLToPath(
+  new URL("../assets/code-quality-criteria.yaml", import.meta.url),
+);
+
+const VALID_SEVERITIES: readonly string[] = Object.values(CodeQualitySeverity);
+
+/** Parse + validate the rubric artifact. PURE given the file; THROWS on any defect. */
+export function parseCodeQualityCriteria(raw: string, source = CODE_QUALITY_CRITERIA_PATH): readonly CodeQualityCriterion[] {
+  // NOTE: a function DECLARATION (not a const arrow) so TS narrows through the
+  // never-returning calls below — control-flow analysis requires this form.
+  function bad(msg: string): never {
+    throw new Error(`code-quality rubric invalid (${source}): ${msg}`);
+  }
+  const doc = parseYaml(raw) as { version?: unknown; criteria?: unknown } | null;
+  if (doc === null || typeof doc !== "object") bad("not a YAML mapping");
+  if (typeof doc.version !== "number") bad("`version` must be a number");
+  if (!Array.isArray(doc.criteria) || doc.criteria.length === 0) bad("`criteria` must be a non-empty list");
+
+  const seen = new Set<string>();
+  const out = (doc.criteria as unknown[]).map((c, i) => {
+    const at = `criteria[${i}]`;
+    if (c === null || typeof c !== "object") return bad(`${at} is not a mapping`);
+    const r = c as Record<string, unknown>;
+    for (const k of ["id", "statement", "passDefinition", "failDefinition", "severity"]) {
+      if (typeof r[k] !== "string" || (r[k] as string).trim() === "") bad(`${at}.${k} must be a non-empty string`);
+    }
+    const id = r.id as string;
+    if (seen.has(id)) bad(`${at}.id "${id}" is duplicated`);
+    seen.add(id);
+    if (!VALID_SEVERITIES.includes(r.severity as string)) {
+      bad(`${at}.severity "${String(r.severity)}" must be one of ${VALID_SEVERITIES.join(" | ")}`);
+    }
+    return {
+      id,
+      statement: r.statement as string,
+      passDefinition: r.passDefinition as string,
+      failDefinition: r.failDefinition as string,
+      severity: r.severity as CodeQualitySeverityValue,
+    } satisfies CodeQualityCriterion;
+  });
+  return Object.freeze(out);
+}
+
+export const DEFAULT_CODE_QUALITY_CRITERIA: readonly CodeQualityCriterion[] =
+  parseCodeQualityCriteria(readFileSync(CODE_QUALITY_CRITERIA_PATH, "utf8"));
 
 /** The code subject under quality review (minimal, subject-agnostic). */
 export interface CodeQualitySubject {

package/.claude/skills/mutagent-evaluator/scripts/config/schema.ts

@@ -235,9 +235,9 @@ export type GlobalSource = Static<typeof GlobalSourceSchema>;
 
 /**
  * A precise link to an Agent / Tooling definition file in a code target.
- * RESERVED — consumed by Build/Improve (future). The evaluator is JUDGE-ONLY (EV-051)
+ * RESERVED — consumed by Build/Optimize (future). The evaluator is JUDGE-ONLY (EV-051)
  * and never writes a target; `targets[].code_refs` is ported for shape parity only and
- * has NO evaluator consumer today. PURPOSE (future): tells BUILD/IMPROVE which files
+ * has NO evaluator consumer today. PURPOSE (future): tells BUILD/OPTIMIZE which files
  * realize the agent so a fix lands on the right def.
  */
 export const CodeRefSchema = Type.Object(
@@ -253,7 +253,7 @@ export type CodeRef = Static<typeof CodeRefSchema>;
  * HOW a fix is applied for a target. `kind` gates the report-only branch.
  * CONSUMER: the diagnostics APPLY path (not the evaluator — judge-only). PURPOSE:
  * selects the write mechanism for a fix.
- *   versioning / pr — RESERVED, consumed by Build/Improve (future). No evaluator
+ *   versioning / pr — RESERVED, consumed by Build/Optimize (future). No evaluator
  *                     consumer; ported for parity. PURPOSE (future): toggle spec-bump /
  *                     open-a-PR on apply.
  */

package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts

@@ -1,84 +1,146 @@
 /**
- * scripts/contracts/agentspec-evals.ts — the MINIMAL agentspec.evals slice the
+ * scripts/contracts/agentspec-evals.ts — the MINIMAL `spec.evaluation` slice the
  * EVAL stage consumes (standalone — NEVER imports the agentspec skill's schema).
  * ---------------------------------------------------------------------------
- * The ADL `*build` stage emits an agentspec whose `definition.evals` block carries
- * the SEED material the evaluator materializes into a real dataset (F8) + criteria:
+ * AgentSpec 0.3.0 (D18/D19) replaces the flat 0.2 `definition.evals` shape
+ * (`dataset_categories[] × edge_cases[]`) with `spec.evaluation`, a universal
+ * closure of THREE arrays (each may be empty, F05):
  *
- *   - dataset_categories[] — `{ id, description, edge_cases[] }` — the golden
- *     eval-suite slices + the explicit edge_cases each must exercise (the dataset
- *     DEFINITION the spec hands forward; "seed, don't duplicate").
- *   - scenarios[]          — `{ id, description, expected_behavior, category?,
- *     edge_case? }` — representative situations + the correct behavior (extra seed
- *     material, optionally tagged into a category).
- *   - success_criteria[]   — `{ id, criterion, type, goal }` — binary-actionable
- *     pass/fail criteria (type ∈ llm-judge | code-check).
+ *   - criteria[]  — `{ id, description, type, goal }` — binary-actionable pass/fail
+ *     criteria (type ∈ llm-judge | code-check). (0.2 `success_criteria[]`, with
+ *     `criterion` → `description`.)
+ *   - scenarios[] — `{ id, description, expectedBehavior, edgeCase? }` — representative
+ *     situations + the correct behavior. Scenario↔dataset linkage now lives on the
+ *     dataset ITEM (`scenarioRef`), not on the scenario.
+ *   - datasets[]  — `{ id, description, mapsTo?, categories[], caseDimensions?,
+ *     items[]|itemsRef? }` — REAL, already-materialized dataset rows (D18):
+ *       · mapsTo        — direct job/scenario/criterion references this dataset exercises.
+ *       · categories[]  — DATASET-LOCAL taxonomy (`{ id, description, generationGuidance? }`),
+ *         no longer a top-level global list (D18).
+ *       · caseDimensions — a MAP `name → { description?, values[] }`; the independent
+ *         variation axes each item's `case` assigns over.
+ *       · items[]       — each `{ id, category?, scenarioRef?, case?, input?, expected? }`
+ *         carrying a KIND-SPECIFIC `input`/`expected` payload (opaque data — we read the
+ *         shape structurally, never interpret the kind).
  *
  * We DECLARE only the fields we read, with `additionalProperties: true` on the
- * objects so a richer agentspec still parses (forward-compatible). The cross-skill
+ * objects so a richer 0.3.x agentspec still parses (forward-compatible). The cross-skill
  * import ban (coding-rules "Sealed-Sibling" + standalone discipline) is why this is
  * a local re-declaration, not an import. PURE — no clock / random / network.
  */
 import { type Static, Type } from "@sinclair/typebox";
 import { Value } from "@sinclair/typebox/value";
 
-/** A dataset CATEGORY slice + the edge-cases it must exercise. */
-export const AgentspecDatasetCategorySchema = Type.Object(
+/** A binary-actionable SUCCESS CRITERION (0.3 `criteria[]`). */
+export const SpecEvalCriterionSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
     description: Type.String({ minLength: 1 }),
-    edge_cases: Type.Array(Type.String()),
+    type: Type.Union([Type.Literal("llm-judge"), Type.Literal("code-check")]),
+    goal: Type.String({ minLength: 1 }),
   },
   { additionalProperties: true },
 );
-export type AgentspecDatasetCategory = Static<typeof AgentspecDatasetCategorySchema>;
+export type SpecEvalCriterion = Static<typeof SpecEvalCriterionSchema>;
 
 /** A representative SCENARIO (the situation + correct behavior). */
-export const AgentspecScenarioSchema = Type.Object(
+export const SpecEvalScenarioSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
     description: Type.String({ minLength: 1 }),
-    expected_behavior: Type.String({ minLength: 1 }),
+    expectedBehavior: Type.String({ minLength: 1 }),
+    edgeCase: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalScenario = Static<typeof SpecEvalScenarioSchema>;
+
+/** A DATASET-LOCAL category slice (D18) — the taxonomy is owned by its dataset. */
+export const SpecEvalDatasetCategorySchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    generationGuidance: Type.Optional(Type.String()),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalDatasetCategory = Static<typeof SpecEvalDatasetCategorySchema>;
+
+/** One independent variation AXIS in a dataset's `caseDimensions` map (the map VALUE). */
+export const SpecEvalCaseDimensionSchema = Type.Object(
+  {
+    description: Type.Optional(Type.String()),
+    values: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalCaseDimension = Static<typeof SpecEvalCaseDimensionSchema>;
+
+/**
+ * A single, already-materialized dataset ITEM. `case` assigns a value per
+ * `caseDimensions` axis; `input`/`expected` are opaque kind-specific payloads we
+ * carry structurally but never interpret.
+ */
+export const SpecEvalDatasetItemSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
     category: Type.Optional(Type.String()),
-    edge_case: Type.Optional(Type.Boolean()),
+    scenarioRef: Type.Optional(Type.String()),
+    case: Type.Optional(Type.Record(Type.String({ minLength: 1 }), Type.String())),
+    input: Type.Optional(Type.Unknown()),
+    expected: Type.Optional(Type.Unknown()),
   },
   { additionalProperties: true },
 );
-export type AgentspecScenario = Static<typeof AgentspecScenarioSchema>;
+export type SpecEvalDatasetItem = Static<typeof SpecEvalDatasetItemSchema>;
 
-/** A binary-actionable SUCCESS CRITERION. */
-export const AgentspecSuccessCriterionSchema = Type.Object(
+/** Direct job/scenario/criterion references a dataset exercises (all optional). */
+export const SpecEvalDatasetMapsToSchema = Type.Object(
+  {
+    jobs: Type.Optional(Type.Array(Type.String())),
+    scenarios: Type.Optional(Type.Array(Type.String())),
+    criteria: Type.Optional(Type.Array(Type.String())),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalDatasetMapsTo = Static<typeof SpecEvalDatasetMapsToSchema>;
+
+/** A dataset: local categories + variation axes + the real materialized `items[]`. */
+export const SpecEvalDatasetSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
-    criterion: Type.String({ minLength: 1 }),
-    type: Type.Union([Type.Literal("llm-judge"), Type.Literal("code-check")]),
-    goal: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    mapsTo: Type.Optional(SpecEvalDatasetMapsToSchema),
+    categories: Type.Optional(Type.Array(SpecEvalDatasetCategorySchema)),
+    caseDimensions: Type.Optional(Type.Record(Type.String({ minLength: 1 }), SpecEvalCaseDimensionSchema)),
+    items: Type.Optional(Type.Array(SpecEvalDatasetItemSchema)),
+    itemsRef: Type.Optional(Type.String()),
   },
   { additionalProperties: true },
 );
-export type AgentspecSuccessCriterion = Static<typeof AgentspecSuccessCriterionSchema>;
+export type SpecEvalDataset = Static<typeof SpecEvalDatasetSchema>;
 
-/** The `definition.evals` slice. Each array may be empty structurally. */
-export const AgentspecEvalsSchema = Type.Object(
+/** The `spec.evaluation` closure. Each array is required but may be empty (F05). */
+export const SpecEvaluationSchema = Type.Object(
   {
-    success_criteria: Type.Array(AgentspecSuccessCriterionSchema),
-    scenarios: Type.Array(AgentspecScenarioSchema),
-    dataset_categories: Type.Array(AgentspecDatasetCategorySchema),
+    criteria: Type.Array(SpecEvalCriterionSchema),
+    scenarios: Type.Array(SpecEvalScenarioSchema),
+    datasets: Type.Array(SpecEvalDatasetSchema),
   },
   { additionalProperties: true },
 );
-export type AgentspecEvals = Static<typeof AgentspecEvalsSchema>;
+export type SpecEvaluation = Static<typeof SpecEvaluationSchema>;
 
 /**
- * Parse + narrow the agentspec.evals slice (guarded). THROWS on schema violation —
- * a malformed seed must never silently reach materialization. PURE.
+ * Parse + narrow the `spec.evaluation` slice (guarded). THROWS on schema violation —
+ * a malformed contract must never silently reach materialization. PURE.
  */
-export function parseAgentspecEvals(value: unknown): AgentspecEvals {
-  if (!Value.Check(AgentspecEvalsSchema, value)) {
-    const first = [...Value.Errors(AgentspecEvalsSchema, value)][0];
+export function parseSpecEvaluation(value: unknown): SpecEvaluation {
+  if (!Value.Check(SpecEvaluationSchema, value)) {
+    const first = [...Value.Errors(SpecEvaluationSchema, value)][0];
     throw new Error(
-      `parseAgentspecEvals: schema violation at '${first?.path ?? "(root)"}': ` +
-        `${first?.message ?? "invalid agentspec evals slice"}`,
+      `parseSpecEvaluation: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid spec.evaluation slice"}`,
     );
   }
   return value;