@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts

@@ -85,6 +85,18 @@ function transcriptOf(trace: EvalTrace): TranscriptTurn[] {
   return turns;
 }
 
+/**
+ * The VERIFY-SURFACE view of a situation: each trace augmented with the SAME
+ * deterministic `trajectory`/`transcript` projections its packet was built from
+ * (`trajectoryOf`/`transcriptOf`). A judge cites refs against the packet it was
+ * handed — those paths must re-resolve at verify time, or a syntax/shape gap
+ * (not evidence weakness) silently downgrades a grounded verdict. Trace-shaped
+ * paths (`observations.N.…`) keep resolving via the spread. PURE.
+ */
+export function verifySituationViews(traces: EvalTrace[]): EvalTrace[] {
+  return traces.map((t) => ({ ...t, trajectory: trajectoryOf(t), transcript: transcriptOf(t) }) as EvalTrace);
+}
+
 // ── T1 (B-U3) tier-0 deterministic pre-pass ──────────────────────────────────
 //
 // THE real cost lever (§9.4.1): run the code-method rows FIRST (deterministic,
@@ -338,6 +350,7 @@ export function buildMatrixPacket(
   criteria: MatrixCriterion[],
   pin: { model: string; temperature: 0 },
   subjectProfile?: SubjectProfile,
+  layerScope?: string[],
 ): MatrixPacket {
   const packet: MatrixPacket = {
     subject,
@@ -349,6 +362,9 @@ export function buildMatrixPacket(
     // byte-stable legacy packet (the judge reconstructs the profile at reason-time).
     ...(subjectProfile !== undefined ? { subjectProfile } : {}),
     pin,
+    // v3 D1=B+ — the pipeable `*evaluate-<layer>` filter. ABSENT/empty ⇒ the FULL
+    // walk (byte-stable legacy packet — the field is omitted entirely).
+    ...(layerScope !== undefined && layerScope.length > 0 ? { layerScope } : {}),
   };
   assertMatrixPacket(packet);
   return packet;
@@ -604,7 +620,14 @@ export function aggregateMatrixScorecard(input: MatrixAggregateInput): MatrixAgg
   // proof that a DIFFERENT judge ran, not just "eligible for".
   const independentVerify: IndependentVerifyRecord[] = [];
   if (input.independentVerify !== undefined) {
-    const { situation, signals } = input.independentVerify;
+    const { situation: rawSituation, signals } = input.independentVerify;
+    // The judge cites refs against the PACKET it was handed (trajectory[]/transcript[]
+    // projections), but the raw EvalTrace carries neither field — so a deterministic
+    // re-resolve against bare traces reads every packet-shaped ref as DEAD and
+    // silently downgrades a grounded fail to uncertain (observed live: gate flipped
+    // fail → incomplete on a verifier-UPHELD defect). Re-resolve against the trace
+    // AUGMENTED with the same deterministic projections the packet was built from.
+    const situation = verifySituationViews(rawSituation);
     const gatingFails = gatingFailVerdicts(verdicts, severityById, input.gatingSeverities);
     const gatingFailIds = new Set(gatingFails.map((v) => v.criterionId));
     for (const v of verdicts) {
@@ -748,3 +771,198 @@ export function readMatrixVerdictFiles(verdictDir: string, trajectoryIds: string
   }
   return raws;
 }
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// v3 LAYERED WALK — aggregate-side folds (E-NEW-1/3/9 · D2-A). ALL PURE, ALL
+// ADDITIVE: consumed by run-evaluate AFTER the (untouched) criteria aggregate;
+// the GATE function is NOT edited — layer verdicts are diagnostic columns and
+// cross-layer CONFLICTS are surfaced for a human, never auto-resolved (OT-1).
+// ═══════════════════════════════════════════════════════════════════════════════
+
+import { LAYER_IDS, type LayerVerdict, type WalkEmissions } from "./contracts/eval-matrix.ts";
+
+/** One trajectory's layer row for the report's layer matrix (skipped ≠ undecidable). */
+export interface LayerFoldRow {
+  trajectoryId: string;
+  /** layer id → verdict; layers the walk never mentioned are `absent` (pre-v3 file →
+   *  NAMED degrade); `fired` = the L0 evidence-state (patterns fired, no verdict). */
+  byLayer: Record<string, "pass" | "fail" | "undecidable" | "skipped" | "fired" | "no-fires" | "absent">;
+  earlyExit?: string;
+  /** COERCED to a count (a live judge may emit the engaged layer-id LIST). */
+  layersEngaged?: number;
+}
+
+/** Per-layer fold buckets. `other` is the NaN-proof catch-all for any tolerant
+ *  verdict value the buckets don't know yet — surfaced, never silently dropped
+ *  (the raw value stays visible in the row's `byLayer`). */
+export interface LayerFoldTotals {
+  pass: number;
+  fail: number;
+  undecidable: number;
+  skipped: number;
+  fired: number;
+  /** L0 clean-state (library ran, no pattern fired) — honest clean, never a pass. */
+  "no-fires": number;
+  absent: number;
+  other: number;
+}
+
+/** Coerce a tolerant `layersEngaged` (count OR layer-id list) to a count. */
+export function layersEngagedCount(v: number | string[] | undefined): number | undefined {
+  return Array.isArray(v) ? v.length : v;
+}
+
+/** E-NEW-1 — fold per-walk layerVerdicts into per-trajectory rows + per-layer totals. */
+export function foldLayerVerdicts(files: MatrixVerdictFile[]): {
+  rows: LayerFoldRow[];
+  totals: Record<string, LayerFoldTotals>;
+} {
+  const totals: Record<string, LayerFoldTotals> = {};
+  for (const layer of LAYER_IDS)
+    totals[layer] = { pass: 0, fail: 0, undecidable: 0, skipped: 0, fired: 0, "no-fires": 0, absent: 0, other: 0 };
+  const rows: LayerFoldRow[] = files.map((file) => {
+    const byLayer: LayerFoldRow["byLayer"] = {};
+    for (const layer of LAYER_IDS) {
+      const lv = (file.layerVerdicts ?? []).find((x: LayerVerdict) => x.layer === layer);
+      const v = (lv?.verdict ?? "absent") as LayerFoldRow["byLayer"][string];
+      byLayer[layer] = v;
+      const bucket = totals[layer]![v as keyof LayerFoldTotals] !== undefined ? (v as keyof LayerFoldTotals) : "other";
+      totals[layer]![bucket] += 1;
+    }
+    return {
+      trajectoryId: file.trajectoryId,
+      byLayer,
+      earlyExit: file.earlyExit,
+      layersEngaged: layersEngagedCount(file.layersEngaged),
+    };
+  });
+  return { rows, totals };
+}
+
+/** E-NEW-3 · D2-A — one detected cross-layer disagreement. NEVER auto-resolved. */
+export interface LayerConflict {
+  trajectoryId: string;
+  /** the disagreeing layers, e.g. ["L1","L2"]. */
+  layers: string[];
+  verdicts: Record<string, string>;
+  /** the judge's own note if it flagged the tension (honest emission). */
+  note?: string;
+}
+
+/**
+ * Detect outcome-vs-deep-layer disagreements (the OT-1 class): L1 pass with any
+ * of L2/L3/L4 fail, or L1 fail with ALL deep layers pass. Detection ONLY — the
+ * conflict routes to the human calibration queue; the GATE never reads it.
+ */
+export function detectLayerConflicts(files: MatrixVerdictFile[]): LayerConflict[] {
+  const out: LayerConflict[] = [];
+  for (const file of files) {
+    const lvs = file.layerVerdicts ?? [];
+    const get = (layer: string): LayerVerdict | undefined => lvs.find((x: LayerVerdict) => x.layer === layer);
+    const l1 = get("L1");
+    if (!l1) continue;
+    const deep = ["L2", "L3", "L4"].map((l) => get(l)).filter((x): x is LayerVerdict => x !== undefined);
+    const deepFails = deep.filter((x) => x.verdict === "fail");
+    const deepDecided = deep.filter((x) => x.verdict === "pass" || x.verdict === "fail");
+    if (l1.verdict === "pass" && deepFails.length > 0) {
+      const layers = ["L1", ...deepFails.map((x) => x.layer)];
+      out.push({
+        trajectoryId: file.trajectoryId,
+        layers,
+        verdicts: Object.fromEntries([["L1", "pass"], ...deepFails.map((x) => [x.layer, x.verdict])]),
+        note: deepFails.map((x) => x.note).filter(Boolean).join(" · ") || l1.note,
+      });
+    } else if (l1.verdict === "fail" && deepDecided.length > 0 && deepFails.length === 0) {
+      out.push({
+        trajectoryId: file.trajectoryId,
+        layers: ["L1", ...deepDecided.map((x) => x.layer)],
+        verdicts: Object.fromEntries([["L1", "fail"], ...deepDecided.map((x) => [x.layer, x.verdict])]),
+        note: l1.note,
+      });
+    }
+  }
+  return out;
+}
+
+/**
+ * COMPLETENESS LAW (S1.2) — every (criterion × complete-fidelity trajectory) cell
+ * must be accounted for: decided, abstained (uncertain), or explicitly `na` in the
+ * denseMap. A silently-missing cell THROWS — a criterion may never go unverdicted
+ * because nothing bound. INCOMPLETE (fidelity) trajectories are exempt by design.
+ */
+export function assertNoUnverdictedCriterion(criteria: MatrixCriterion[], files: MatrixVerdictFile[]): void {
+  const missing: string[] = [];
+  for (const file of files) {
+    if (file.fidelity && file.fidelity.complete === false) continue; // INCOMPLETE exempt
+    const judged = new Set(file.verdicts.map((v) => v.criterionId));
+    const dense = file.denseMap ?? {};
+    for (const c of criteria) {
+      if (judged.has(c.criterionId)) continue;
+      if (dense[c.criterionId] !== undefined) continue; // na / dense-mapped = accounted
+      missing.push(`${file.trajectoryId} × ${c.criterionId}`);
+    }
+  }
+  if (missing.length > 0) {
+    throw new Error(
+      `COMPLETENESS LAW: ${missing.length} (criterion × trajectory) cell(s) unaccounted — a criterion ` +
+        `may NEVER be silently unverdicted (decide, abstain-with-reason, or mark na in denseMap). ` +
+        `First offenders: ${missing.slice(0, 5).join(" · ")}`,
+    );
+  }
+}
+
+/** MR-5 (E-NEW-9b) — fold the walks' emissions self-manifests into the run
+ *  data-completeness scorecard: expected-key coverage + every NAMED missing reason. */
+export interface EmissionsScorecard {
+  walksWithManifest: number;
+  walksTotal: number;
+  /** expected emission keys of the v3 contract. */
+  expectedKeys: string[];
+  /** key → number of complete-fidelity walks that emitted it. */
+  emittedCounts: Record<string, number>;
+  /** every named degrade: {trajectoryId, key, reason}. */
+  namedMissing: { trajectoryId: string; key: string; reason: string }[];
+  /** walks that emitted NO manifest at all (pre-v3 / defect — a named degrade itself). */
+  manifestAbsent: string[];
+}
+
+export const V3_EXPECTED_EMISSIONS: readonly string[] = [
+  "layerVerdicts",
+  "understanding",
+  "expectedTrajectory",
+  "agentSteps",
+  "judgeSteps",
+  "emissions",
+];
+
+export function foldEmissionsScorecard(files: MatrixVerdictFile[]): EmissionsScorecard {
+  const complete = files.filter((f) => !(f.fidelity && f.fidelity.complete === false));
+  const emittedCounts: Record<string, number> = {};
+  for (const k of V3_EXPECTED_EMISSIONS) emittedCounts[k] = 0;
+  const namedMissing: EmissionsScorecard["namedMissing"] = [];
+  const manifestAbsent: string[] = [];
+  for (const f of complete) {
+    // ground truth from the file itself (never trust-only the self-report):
+    const present = (k: string): boolean =>
+      (f as unknown as Record<string, unknown>)[k] !== undefined &&
+      (!Array.isArray((f as unknown as Record<string, unknown>)[k]) ||
+        ((f as unknown as Record<string, unknown>)[k] as unknown[]).length > 0);
+    for (const k of V3_EXPECTED_EMISSIONS) if (present(k)) emittedCounts[k]! += 1;
+    const manifest: WalkEmissions | undefined = f.emissions;
+    if (!manifest) {
+      manifestAbsent.push(f.trajectoryId);
+      continue;
+    }
+    for (const m of manifest.missing ?? []) {
+      namedMissing.push({ trajectoryId: f.trajectoryId, key: m.key, reason: m.reason });
+    }
+  }
+  return {
+    walksWithManifest: complete.length - manifestAbsent.length,
+    walksTotal: complete.length,
+    expectedKeys: [...V3_EXPECTED_EMISSIONS],
+    emittedCounts,
+    namedMissing,
+    manifestAbsent,
+  };
+}

package/.claude/skills/mutagent-evaluator/scripts/memory/append.ts

@@ -37,7 +37,7 @@ export const Lifecycle = {
   Build: "build",
   Evaluate: "evaluate",
   Diagnose: "diagnose",
-  Improve: "improve",
+  Optimize: "optimize",
   General: "general",
 } as const;
 export type LifecycleValue = (typeof Lifecycle)[keyof typeof Lifecycle];

package/.claude/skills/mutagent-evaluator/scripts/memory/ratify.ts

@@ -0,0 +1,165 @@
+/**
+ * scripts/memory/ratify.ts — EV-4: persist ratify / do-not decisions + hold the verdict.
+ * ---------------------------------------------------------------------------
+ * The dominant dogfood finding ("router-as-doer") rested on a NORMATIVE call —
+ * what counts as correct routing — that only the operator can settle. The review
+ * UI already CAPTURES the operator's verify/eliminate decisions on each surfaced
+ * assumption (`build-review-ui.ts` `setCalib` → `calibration.json`), but wrote them
+ * NOWHERE: a verdict resting on an unratified standard still read as final.
+ *
+ * This is the LEAN wiring (decision EV-4 option a — NO new criterion schema):
+ *   1. ROUTE each calibration decision into `appendMemory` — the same store the
+ *      next run's judges read at start (`memory/read.ts`), deduped by slug.
+ *   2. HOLD the verdict: a NORMATIVE criterion with no `verify` ratification in the
+ *      decision set is `needs-ratification`; while any is pending the run's verdict
+ *      is HELD (not final) — the value call must be settled first.
+ *
+ * PURE cores (deterministic; `now` injected). The only fs edge reuses `appendMemory`.
+ */
+import { appendMemory, Lifecycle, MemoryType, type AppendResult } from "./append.ts";
+import { AssumptionKind, type CriterionVerdict } from "../contracts/eval-types.ts";
+
+/**
+ * The criterion ids whose verdict rests on a NORMATIVE assumption (a value/standard
+ * call — `AssumptionKind.Normative`), whether surfaced in `assumptions[]` or the
+ * `blockedBy` payload. These are the criteria that REQUIRE operator ratification
+ * before their verdict can be final. PURE.
+ */
+export function normativeCriterionIds(verdicts: CriterionVerdict[]): string[] {
+  const ids = new Set<string>();
+  for (const v of verdicts) {
+    const inAssumptions = (v.assumptions ?? []).some((a) => a.kind === AssumptionKind.Normative);
+    const inBlock = v.blockedBy?.kind === AssumptionKind.Normative;
+    if (inAssumptions || inBlock) ids.add(v.criterionId);
+  }
+  return [...ids].sort();
+}
+
+/** The verify/eliminate action a review-UI calibration button emits (`data-action`). */
+export const RatifyAction = {
+  /** the operator RATIFIES the surfaced assumption/standard (accept it). */
+  Verify: "verify",
+  /** the operator rejects it (do-not — drop this standard). */
+  Eliminate: "eliminate",
+} as const;
+export type RatifyActionValue = (typeof RatifyAction)[keyof typeof RatifyAction];
+
+/**
+ * One calibration decision as the review UI exports it (`calibration.json`). Shape
+ * mirrors `build-review-ui.ts` `setCalib` EXACTLY so the operator's export routes
+ * with no transform.
+ */
+export interface CalibrationDecision {
+  traceId: string;
+  criterionId: string;
+  assumptionIndex: number;
+  action: RatifyActionValue;
+  decidedAt?: string;
+}
+
+/** A stable slug for a ratification memory (dedupe: re-deciding the same pair updates). */
+export function ratificationSlug(d: CalibrationDecision): string {
+  return `ratify-${d.criterionId}-a${d.assumptionIndex}`;
+}
+
+/**
+ * Project a calibration decision → an AutoMemory FeedbackInput. A `verify` is a
+ * standing "this standard IS ratified" fact the next run's judge honours; an
+ * `eliminate` records the operator dropped it. PURE.
+ */
+export function calibrationToMemory(
+  d: CalibrationDecision,
+  criterionStatement?: string,
+): { title: string; description: string; body: string } {
+  const ratified = d.action === RatifyAction.Verify;
+  const what = criterionStatement !== undefined && criterionStatement.length > 0 ? criterionStatement : d.criterionId;
+  const title = ratificationSlug(d);
+  const verb = ratified ? "RATIFIED" : "REJECTED (do-not)";
+  return {
+    title,
+    description: `Normative ratification — criterion ${d.criterionId} assumption #${d.assumptionIndex} ${verb}`,
+    body:
+      `Operator ${verb} the normative standard for criterion \`${d.criterionId}\` ` +
+      `(assumption #${d.assumptionIndex}).\n\n` +
+      `**What:** ${what}\n` +
+      `**Why:** a normative criterion (what SHOULD count as good) is an operator value call — ` +
+      `the judge cannot settle it. This decision is the settled standard.\n` +
+      `**How to apply:** ${ratified
+        ? "treat this standard as ratified — the verdict on this criterion may now be final."
+        : "do NOT gate on this standard — the operator dropped it."}\n` +
+      (d.decidedAt !== undefined ? `\n(decided ${d.decidedAt})` : ""),
+  };
+}
+
+/**
+ * ROUTE calibration decisions into AutoMemory (EV-4). One `appendMemory` per
+ * decision, deduped by slug (re-deciding UPDATES). Returns the append results.
+ * `now` (YYYY-MM-DD) is INJECTED. `statements` optionally maps criterionId → its
+ * statement for a richer memory body.
+ */
+export function persistCalibrations(
+  memoryDir: string,
+  decisions: CalibrationDecision[],
+  now: string,
+  statements: Record<string, string> = {},
+): AppendResult[] {
+  return decisions.map((d) => {
+    const m = calibrationToMemory(d, statements[d.criterionId]);
+    return appendMemory(
+      memoryDir,
+      {
+        title: m.title,
+        description: m.description,
+        lifecycle: Lifecycle.Evaluate,
+        type: MemoryType.Feedback, // a standing steer the next run's judge honours
+        feedback: { text: m.body },
+      },
+      now,
+    );
+  });
+}
+
+/** The ratification state of one normative criterion (EV-4). */
+export interface RatificationStatus {
+  criterionId: string;
+  /** true when a `verify` decision ratified it; false ⇒ needs-ratification. */
+  ratified: boolean;
+}
+
+/**
+ * Given the NORMATIVE criterion ids (those resting on a normative assumption) and
+ * the operator's calibration decisions, report which are RATIFIED (a `verify`
+ * exists) vs still `needs-ratification`. An `eliminate` does NOT ratify — the
+ * standard was dropped, not settled-as-correct. PURE.
+ */
+export function ratificationStatus(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): RatificationStatus[] {
+  const ratified = new Set(
+    decisions.filter((d) => d.action === RatifyAction.Verify).map((d) => d.criterionId),
+  );
+  return normativeCriterionIds.map((id) => ({ criterionId: id, ratified: ratified.has(id) }));
+}
+
+/**
+ * The criterion ids that HOLD the verdict (EV-4): normative criteria not yet
+ * ratified. While this list is non-empty the run's verdict is NOT final — the
+ * operator must settle the value call first. PURE.
+ */
+export function pendingRatifications(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): string[] {
+  return ratificationStatus(normativeCriterionIds, decisions)
+    .filter((s) => !s.ratified)
+    .map((s) => s.criterionId);
+}
+
+/** Is the run's verdict HELD pending an operator ratification? (EV-4) */
+export function isVerdictHeldForRatification(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): boolean {
+  return pendingRatifications(normativeCriterionIds, decisions).length > 0;
+}