@mutagent/evaluator 0.1.0-alpha.6 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts

@@ -91,6 +91,15 @@ export const MatrixCriterionSchema = Type.Object(
     ]),
     /** the 3-dimension MECE coverage tag (optional context for the judge). */
     dimension: Type.Optional(Type.String()),
+    /**
+     * WHICH evidence layer this criterion binds to (the v3 layered walk's own
+     * taxonomy): a judge-walk layer id (`"L0".."L4"`), a discover-proposed
+     * MetricLevel (`"output" | "context" | "cross-stage"`), or the criteria
+     * super-layer (`"criteria"`) for a cross-cutting check. ADDITIVE/OPTIONAL —
+     * absent ⇒ renderers default to `"criteria"` (byte-stable pre-v3 rows).
+     * Tolerant free string for forward-compat; consumers map to LAYER_NAMES.
+     */
+    layer: Type.Optional(Type.String()),
     /** what the judge should read for this row (write-judge-prompt "choose what to pass"). */
     judgeInputs: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
     // ── T1 (B-U3) tier-0 router (ADDITIVE, OPTIONAL — grandfather) ──────────────
@@ -138,6 +147,7 @@ export function minedToMatrixCriterion(c: MinedCriterion): MatrixCriterion {
     passCondition: c.statement,
     severity: c.metadata.severity,
     dimension: c.metadata.dimension,
+    layer: c.metadata.level,
     checkMethod: c.metadata.check_method,
     ...(judgeInputs.length > 0 ? { judgeInputs } : {}),
     ...(c.codeEval !== undefined ? { codeEval: c.codeEval } : {}),
@@ -281,6 +291,14 @@ export const MatrixPacketSchema = Type.Object(
       { model: Type.String({ minLength: 1 }), temperature: Type.Literal(0) },
       { additionalProperties: false },
     ),
+    /**
+     * v3 LAYERED WALK (D1=B+) — layer-scope filter for the pipeable
+     * `*evaluate-<layer>` commands: absent/empty = the FULL walk (L0→L4 +
+     * criteria); a subset = the judge runs ONLY those layer phases (criteria
+     * still verdicted over whatever those phases gather — never left
+     * unverdicted). ADDITIVE/OPTIONAL: legacy packets still validate.
+     */
+    layerScope: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
   },
   { additionalProperties: false },
 );
@@ -392,7 +410,9 @@ export type ExpectedStep = Static<typeof ExpectedStepSchema>;
 export const AgentStepSchema = Type.Object(
   {
     n: Type.Number(),
-    tool: Type.Optional(Type.String()),
+    /** TOLERANT: explicit `null` = an honest "no tool call this step" (live-judge shape
+     *  on a zero-tool trajectory) — distinct from an omitted field. */
+    tool: Type.Optional(Type.Union([Type.String(), Type.Null()])),
     /** ok | error | warn | false-success */
     status: Type.Optional(Type.String()),
     detail: Type.Optional(Type.String()),
@@ -448,6 +468,139 @@ export const CandidateItemSchema = Type.Object(
 );
 export type CandidateItem = Static<typeof CandidateItemSchema>;
 
+// ── v3 LAYERED WALK (E-NEW-1..6,9 — ADDITIVE, OPTIONAL) ──────────────────────
+// Evaluator-v3 contract (operator-frozen 2026-07-24): ONE judge per trajectory
+// performs the whole evidence-layer walk internally — L0 code checks → L1
+// outcome → (descend per stopping rules) L2 trajectory → L3 tool outputs → L4
+// context — and EMITS one LayerVerdict per engaged layer alongside the
+// (unchanged) per-criterion verdicts. Cross-layer disagreement is SIGNAL:
+// aggregate DETECTS conflicts, never resolves them (D2-A / OT-1). All fields
+// OPTIONAL (grandfather): pre-v3 verdict files still validate; renderers show
+// NAMED degrade badges, never silent blanks.
+
+/** The five evidence layers of the walk. */
+export const LayerId = {
+  Code: "L0",
+  Outcome: "L1",
+  Trajectory: "L2",
+  ToolOutputs: "L3",
+  Context: "L4",
+} as const;
+export type LayerIdValue = (typeof LayerId)[keyof typeof LayerId];
+export const LAYER_IDS: readonly string[] = Object.values(LayerId);
+
+/** Human names — the vocabulary reports print beside the L-codes. */
+export const LAYER_NAMES: Record<string, string> = {
+  L0: "code checks",
+  L1: "outcome",
+  L2: "trajectory",
+  L3: "tool outputs",
+  L4: "context",
+};
+
+/** One per-layer verdict from the walk (E-NEW-1). `skipped` marks layers the
+ *  stopping rules never engaged (early-exit) — skipped ≠ undecidable. */
+export const LayerVerdictSchema = Type.Object(
+  {
+    /** L0 | L1 | L2 | L3 | L4 (tolerant free string for forward-compat). */
+    layer: Type.String({ minLength: 1 }),
+    verdict: Type.Union([
+      Type.Literal("pass"),
+      Type.Literal("fail"),
+      Type.Literal("undecidable"),
+      Type.Literal("skipped"),
+      /** L0 EVIDENCE-STATE (live-judge shape): patterns fired, evidence recorded —
+       *  honoring the meta-judge fence a pass/fail on L0 would violate (a fired
+       *  pattern is a candidate, not a verdict). L0-specific. */
+      Type.Literal("fired"),
+      /** L0 CLEAN-STATE (live-judge shape): the library RAN and NO pattern fired.
+       *  Distinct from `pass` (a pass would assert L0 proved the layer correct) and
+       *  from `skipped` (the library did run). "no code-detectable candidate" — an
+       *  honest clean, never a verdict on the deeper layers. */
+      Type.Literal("no-fires"),
+    ]),
+    /** annotation-only — may NEVER flip the verdict. */
+    confidence: Type.Optional(Type.Number()),
+    /** grounding citations (same tolerant shape as judge-step refs). */
+    refs: Type.Optional(Type.Array(Type.Union([Type.String(), GaRefSchema]))),
+    /** one-breath reasoning for THIS layer's verdict. */
+    note: Type.Optional(Type.String()),
+    /** L1: the classified scenario (STRUCTURED, not prose — E-NEW-2). */
+    scenario: Type.Optional(Type.String()),
+    /** L1: the expected terminal actions for that scenario (E-NEW-2). */
+    expectedExit: Type.Optional(Type.Array(Type.String())),
+    /** L3: the first problematic observation — everything downstream suspect. */
+    /** TOLERANT: explicit `null` = an honest "no problem origin exists" (live-judge
+     *  shape — naming one would invent it), distinct from an omitted field. */
+    firstProblemOrigin: Type.Optional(Type.Union([Type.String(), Type.Null()])),
+    /**
+     * L4: retrieval (never fetched) vs attention (present but ignored).
+     *
+     * TOLERANT-BUT-STILL-CLASSIFIED. A live judge emitted the required distinction
+     * and then QUALIFIED it — "attention (transient, non-terminal — self-corrected
+     * before send); no retrieval gap" — which the bare 2-literal union rejected.
+     * Both forms are now accepted, but deliberately NOT by loosening to a free
+     * string: the qualified form must still BEGIN with one of the two kinds, so
+     * the classification stays machine-extractable (read the leading token) and a
+     * value that classifies nothing (`"unclear"`) is still refused. Loosening to
+     * `Type.String()` would have kept this file parsing while silently forfeiting
+     * the fail-loud property the field exists for.
+     */
+    gapKind: Type.Optional(
+      Type.Union([
+        Type.Literal("retrieval"),
+        Type.Literal("attention"),
+        Type.String({ pattern: "^(retrieval|attention)\\b" }),
+      ]),
+    ),
+    /**
+     * L2: the ONE-LINE statement of where the actual path left the expected path
+     * — the headline of a trajectory-layer failure, distilled out of the much
+     * longer `note`. A live judge emitted it alongside a `note` that walks the
+     * full chronological path, i.e. it is the summary a reader needs, not a
+     * duplicate: "mandatory reviewWithManager step omitted between draft
+     * generation and the terminal sendMessage".
+     */
+    divergence: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type LayerVerdict = Static<typeof LayerVerdictSchema>;
+
+/** E-NEW-6 — one L0 code-eval library hit (the judge runs the library itself). */
+export const CodeEvalHitSchema = Type.Object(
+  {
+    /** the library pattern id (e.g. CE-P1 fault-passthrough). */
+    pattern: Type.String({ minLength: 1 }),
+    /** where it fired (obs id / step anchor). */
+    anchor: Type.Optional(Type.Union([Type.Number(), Type.String()])),
+    detail: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type CodeEvalHit = Static<typeof CodeEvalHitSchema>;
+
+/** E-NEW-9b — the walk's SELF-MANIFEST of emitted vs missing emissions; AGGREGATE
+ *  folds these into the MR-5 data-completeness scorecard so a skipped expected
+ *  emission is caught the same run, not at render time. */
+export const WalkEmissionsSchema = Type.Object(
+  {
+    /** emission keys the judge DID produce. */
+    emitted: Type.Array(Type.String()),
+    /** expected-but-missing keys, each with the judge's own reason. */
+    missing: Type.Optional(
+      Type.Array(
+        Type.Object(
+          { key: Type.String({ minLength: 1 }), reason: Type.String({ minLength: 1 }) },
+          { additionalProperties: false },
+        ),
+      ),
+    ),
+  },
+  { additionalProperties: false },
+);
+export type WalkEmissions = Static<typeof WalkEmissionsSchema>;
+
 // ── MatrixVerdictFile — the Judge Agent OUTPUT (one per trajectory) ──────────
 export const MatrixVerdictFileSchema = Type.Object(
   {
@@ -508,10 +661,91 @@ export const MatrixVerdictFileSchema = Type.Object(
         Type.String(),
         Type.Object(
           {
-            root: Type.String({ minLength: 1 }),
+            /** The primary root (root-not-symptom). REQUIRED on a structured localize —
+             *  a rootless walk emits `null` (an honest "no failure to localize"), and a
+             *  judge that has only a `summary` has its root FILLED from it at parse time
+             *  (see the coercion below) so the invariant holds without rejecting honest
+             *  output. */
+            root: Type.Union([Type.String({ minLength: 1 }), Type.Null()]),
+            /**
+             * A short SUMMARY a judge emits when there is NO failure to localize to a
+             * single root (a clean/hold trajectory with independent non-criterion
+             * deviations). Live-judge shape: instead of `root`, it summarises the whole
+             * trajectory and routes the deviations to `candidates`. Consumers coerce
+             * either to a readable band (`root` preferred, `summary` fallback).
+             */
+            summary: Type.Optional(Type.String()),
             symptom: Type.Optional(Type.String()),
+            /** grounding prose for the root claim (first emitted by a live D1 judge). */
+            evidence: Type.Optional(Type.String()),
+            /** the criterionIds this root OWNS (judge-emitted consolidate-by-locus hint). */
+            criteria: Type.Optional(Type.Array(Type.String())),
+            note: Type.Optional(Type.String()),
             ref: Type.Optional(Type.Union([Type.String(), GaRefSchema])),
             routeTo: Type.Optional(Type.String()),
+            /**
+             * FURTHER roots that are INDEPENDENT of `root` — not steps in one causal
+             * chain, but separate failures that happen to appear in the same
+             * trajectory. Emitted by a live judge following `root_not_symptom`'s
+             * "multiple INDEPENDENT roots ⇒ multiple criteria": it kept the primary
+             * root in `root` and listed the rest here rather than silently collapsing
+             * them into one locus. Consolidate-by-locus clusters on `root`; these are
+             * additional loci, so a consumer that reads only `root` UNDER-REPORTS.
+             */
+            independentRoots: Type.Optional(Type.Array(Type.String())),
+            /**
+             * The L2 actual-vs-expected divergence. Accepted HERE as well as on the
+             * layer verdict because the OPERATOR-FROZEN judge prose explicitly says
+             * "(L2 trajectory) … emit layerVerdicts L2 (+ divergence in localize)" —
+             * so a judge obeying the contract literally puts it in this object, and
+             * the closed schema rejected exactly that. (The live D1 judge happened to
+             * put it on `layerVerdicts[L2]` instead, which is why this never fired.)
+             * Same class as the `rootCause`/`firstThingWrong` drift: contract prose
+             * naming a field the schema refuses. The prose is C-PIN golden, so the
+             * SCHEMA accepts both placements rather than churning a frozen prompt.
+             */
+            divergence: Type.Optional(Type.String()),
+            /**
+             * An OT-1 conflict the judge surfaced UNRECONCILED (e.g. the outcome layer
+             * passes while the trajectory layer fails). Doctrine is to expose the
+             * disagreement for a human rather than have the judge resolve it, so this
+             * is prose stating both sides and explicitly declining to reconcile —
+             * frequently the single most decision-relevant field on the file.
+             */
+            /**
+             * An OT-1 conflict the judge surfaced UNRECONCILED (e.g. the outcome layer
+             * passes while the trajectory layer fails). Doctrine is to expose the
+             * disagreement for a human rather than have the judge resolve it. TOLERANT:
+             * free prose, OR a structured {sideA, sideB, note?} pair (a live judge
+             * emitted the structured form; both carry the same meaning — two sides,
+             * explicitly not reconciled). Frequently the single most decision-relevant
+             * field on the file.
+             */
+            conflict: Type.Optional(
+              Type.Union([
+                Type.String(),
+                Type.Object(
+                  {
+                    sideA: Type.String(),
+                    sideB: Type.String(),
+                    note: Type.Optional(Type.String()),
+                  },
+                  { additionalProperties: false },
+                ),
+                /** `null` = an honest "no conflict — the layers agree" (live-judge shape). */
+                Type.Null(),
+              ]),
+            ),
+            /**
+             * EV-051 routing prose. NOTE: this is the SAME CONCEPT as `routeTo` above,
+             * emitted under a second name by a real judge. Both are accepted because
+             * rejecting the file over a synonym is the worse failure — but do NOT add
+             * a third spelling: a closed schema carrying synonyms is how the
+             * `rootCause`/`firstThingWrong` drift started. `routeTo` currently has ZERO
+             * consumers, so consolidating onto ONE of these is a free cleanup whenever
+             * someone touches the routing path.
+             */
+            routing: Type.Optional(Type.String()),
           },
           { additionalProperties: false },
         ),
@@ -534,6 +768,13 @@ export const MatrixVerdictFileSchema = Type.Object(
         ]),
       ),
     ),
+    /**
+     * E-NEW additive (first emitted by a live D1 judge): per-criterion REASON a
+     * denseMap cell is `na` (`cid → why the scoping predicate is positively
+     * falsified`). `na` is the third accounting path of the completeness law —
+     * this carries its rationale so the report can display WHY, not just that.
+     */
+    naRationale: Type.Optional(Type.Record(Type.String(), Type.String())),
     /**
      * §9.4.2 node 1 — the EXAMINE fidelity gate. `complete:false` means the trace
      * was truncated / unreadable → the trajectory EXITS early as INCOMPLETE (no
@@ -547,6 +788,19 @@ export const MatrixVerdictFileSchema = Type.Object(
     ),
     /** §9.4.2 node 2.5 — candidate eval/dataset items for UNMATCHED detections. */
     candidates: Type.Optional(Type.Array(CandidateItemSchema)),
+    // ── v3 LAYERED WALK emissions (E-NEW — OPTIONAL/additive, pre-v3 files validate) ──
+    /** E-NEW-1 — one verdict per engaged evidence layer (skipped ≠ undecidable). */
+    layerVerdicts: Type.Optional(Type.Array(LayerVerdictSchema)),
+    /** E-NEW-1 — which layer (if any) the stopping rules exited at (e.g. "L1"). */
+    earlyExit: Type.Optional(Type.String()),
+    /** E-NEW-1/9 — how many layers the walk actually engaged (economy stat).
+     *  TOLERANT: a count OR the engaged layer-id list (live-judge shape,
+     *  e.g. ["L0","L1","L2"] — richer; consumers coerce via length). */
+    layersEngaged: Type.Optional(Type.Union([Type.Number(), Type.Array(Type.String())])),
+    /** E-NEW-6 — L0 code-eval library hits the judge recorded (it runs the library itself). */
+    codeEvalHits: Type.Optional(Type.Array(CodeEvalHitSchema)),
+    /** E-NEW-9b — the walk's self-manifest of emitted vs missing emissions (→ MR-5). */
+    emissions: Type.Optional(WalkEmissionsSchema),
     /**
      * the per-trajectory judge SELF-REPORTED health micro-summary. NOTE: this is
      * IGNORED for the report's health roll-up — T4 DERIVES health from the walk
@@ -578,7 +832,9 @@ export type MatrixVerdictFile = Static<typeof MatrixVerdictFileSchema>;
 export function localizeText(localize: MatrixVerdictFile["localize"]): string {
   if (localize === undefined) return "";
   if (typeof localize === "string") return localize;
-  return localize.symptom !== undefined ? `${localize.root} (symptom: ${localize.symptom})` : localize.root;
+  // `null` root = an honest "no failure to localize" — the summary carries the band.
+  const root = localize.root ?? localize.summary ?? "";
+  return localize.symptom !== undefined ? `${root} (symptom: ${localize.symptom})` : root;
 }
 
 /**
@@ -594,6 +850,34 @@ export function parseMatrixVerdictFile(raw: string): MatrixVerdictFile {
   } catch {
     throw new Error(`parseMatrixVerdictFile: not valid JSON: ${raw.slice(0, 120)}`);
   }
+  // COERCE the live-judge rootless-localize: a judge with NO single root to localize
+  // emits a structured localize WITHOUT `root` — carrying a `summary` (clean/hold
+  // trajectory) or a `conflict` (a real disagreement whose disagreement IS the thing to
+  // localize). Rather than weaken the contract's required-root invariant (a guard test
+  // asserts a structured localize with no root is refused), FILL the root from the
+  // summary, or from the conflict's own text, so honest rootless output is accepted AND
+  // the invariant holds. NOTHING is discarded (root takes precedence when present). A
+  // structured localize with NONE of root/summary/conflict still fails — the guard shape.
+  if (
+    parsed !== null &&
+    typeof parsed === "object" &&
+    !Array.isArray(parsed) &&
+    (parsed as Record<string, unknown>)["localize"] !== null &&
+    typeof (parsed as Record<string, unknown>)["localize"] === "object"
+  ) {
+    const loc = (parsed as Record<string, unknown>)["localize"] as Record<string, unknown>;
+    if (loc["root"] === undefined) {
+      if (typeof loc["summary"] === "string") {
+        loc["root"] = loc["summary"];
+      } else if (typeof loc["conflict"] === "string") {
+        loc["root"] = loc["conflict"];
+      } else if (loc["conflict"] !== null && typeof loc["conflict"] === "object") {
+        const c = loc["conflict"] as Record<string, unknown>;
+        const parts = [c["sideA"], c["sideB"], c["note"]].filter((x): x is string => typeof x === "string");
+        if (parts.length > 0) loc["root"] = parts.join(" · ");
+      }
+    }
+  }
   if (!Value.Check(MatrixVerdictFileSchema, parsed)) {
     const first = [...Value.Errors(MatrixVerdictFileSchema, parsed)][0];
     throw new Error(

package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts

@@ -642,6 +642,22 @@ export interface DiscoveryRationale {
 export interface MinedCriterion extends DiscoveredCriterion {
   metadata: MetricMetadata;
   discovery: DiscoveryRationale;
+  /**
+   * The criterion ids ABSORBED into this one by the near-duplicate MERGE
+   * (`scripts/merge-criteria.ts`). Provenance, not a pointer: the absorbed
+   * criterion's evidence lives on THIS criterion (refs unioned, prevalence
+   * recomputed over the union), so the merge is lossless and auditable. The
+   * FIRST-SEEN id survives; every absorbed id is listed here. Absent ⇒ nothing
+   * was merged into this criterion.
+   */
+  mergedFrom?: string[];
+  /**
+   * Alternate ids that RESOLVE to this criterion — the absorbed ids kept live so
+   * an older report, a dataset recipe or a handoff that names the pre-merge id
+   * still finds its criterion (`resolveCriterionId`). A merge must never
+   * invalidate an outstanding reference.
+   */
+  aliases?: string[];
   /**
    * The EXECUTABLE code-check reference (the uniform-standard companion of the
    * human-readable `statement`). REQUIRED when `metadata.check_method` is
@@ -848,6 +864,10 @@ export const MinedCriterionSchema = Type.Object(
     supportCount: Type.Integer({ minimum: 0 }),
     metadata: MetricMetadataSchema,
     discovery: DiscoveryRationaleSchema,
+    /** near-duplicate MERGE provenance — the absorbed ids (first-seen id survives). */
+    mergedFrom: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
+    /** alternate ids that still RESOLVE here (an absorbed id never goes dead). */
+    aliases: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
     /** the executable code-check reference (uniform standard); OPTIONAL at the
      *  schema layer — `lint-uniformity.ts` enforces presence by check_method. */
     codeEval: Type.Optional(CodeEvalSpecSchema),
@@ -987,12 +1007,18 @@ function locateByObs(obs: string, traces: EvalTrace[]): EvalTrace | undefined {
 
 /**
  * Read the value at a dotted `path` (e.g. `output.response`, `observations.0.output`)
- * out of a trace. Numeric segments index arrays. Returns `undefined` when ANY
- * segment is absent (no value at that path) — the caller treats that as
- * UNRESOLVED. An empty path reads the whole trace object. PURE.
+ * out of a trace. Numeric segments index arrays. Bracket indices are TOLERATED
+ * (`observations[3].output` ≡ `observations.3.output` — judges naturally emit the
+ * bracket form; a syntax preference must never manufacture a dead ref). Returns
+ * `undefined` when ANY segment is absent (no value at that path) — the caller
+ * treats that as UNRESOLVED. An empty path reads the whole trace object. PURE.
  */
 function readPath(root: unknown, path: string): unknown {
-  const segments = path.split(".").map((s) => s.trim()).filter((s) => s.length > 0);
+  const segments = path
+    .replace(/\[(\d+)\]/g, ".$1")
+    .split(".")
+    .map((s) => s.trim())
+    .filter((s) => s.length > 0);
   let cur: unknown = root;
   for (const seg of segments) {
     if (cur === null || cur === undefined) return undefined;