@mutagent/evaluator 0.1.0-alpha.4 → 0.1.0-alpha.5

package/.claude/skills/mutagent-evaluator/SKILL.md

@@ -23,7 +23,7 @@ license: Proprietary. LICENSE.txt has complete terms.
 compatibility: Designed for Claude Code, Codex, Cursor, OpenCode and similar coding-agent runtimes; works with git, gh CLI, jq, curl, and Bun/pnpm/npm runtimes.
 metadata:
   author: mutagent
-  version: "0.1.0-alpha.2"
+  version: "0.1.0-alpha.4"
 # allowed-tools: OMITTED — agent uses all native tools per host runtime
 ---
 
@@ -276,6 +276,7 @@ applies, EV-053/EQ5):
 | `scripts/aggregate-discover.ts` | P2 (§5a) + §9.4 (T6) | code | The real `*discover-evals` AGGREGATE half: parse the leaf's per-trace verdict files + per-batch mining report → `TraceAnnotation[]` (FAIL-LOUD via `missingVerdictKeys`; THROW on an undispatched-trace category ref) → `deriveMinedCriteria` (§5b metadata + §5c DR-2) → `growLivingSuite` (append-only, monotonic, saturation-stop). **T6:** `collectDatasetCandidates` (failure/uncertain → `DatasetCase[]` via the derive-dataset selectors) + `unmatchedDetectionCandidates` (the `*evaluate` judge's node-2.5 handoff) → consumable by `*build-dataset`. `DiscoverMiningReport` contract. No hand-rolled annotations. |
 | `scripts/code-eval.ts` | P3 (EX-2) | code | The v2 CODE-track primitive library — generic PURE `(trace) → {result, detail}` checks: presence · string-equality · format-validity · schema-conformance · ref-integrity (cross-stage). SV-1 dotted-path read incl. `obs:<name>.<path>`. Exhaustive `runCodeEval` (unknown primitive → THROW). NO LLM, byte-identical. Disjoint from the v1 `run-deterministic.ts` audit world. |
 | `scripts/check-method-router.ts` | P3 (EX-2) | hybrid | THE load-bearing §5b `check_method` router: `deterministic`→CODE-EXEC (judge NEVER called) · `llm-judge`→JUDGE seam · `hybrid`→code pre-filter that GATES the judge. Unknown → THROW (exhaustive). `RoutedCriterionVerdict` = `CriterionVerdict` + provenance `{producedBy, codeResult, judgeReached}` (structural superset → `evaluate.ts` rollup unchanged). `extractCodeEval(registry)` fail-loud (a code-class criterion with no spec THROWS — never silently judge). "Code before judge" lives INSIDE the agent. |
+| `scripts/code-quality-verdict.ts` | W2I1 (⑤ IMPROVE) | hybrid | The **code-quality JUDGE mode** (`#mode-judge-code-quality`) — the (b) half of the ⑤ IMPROVE code-target BOTH-gate. Scores a `code` subject's QUALITY (spec-faithfulness · addresses-diagnosis · no-regression · error-handling · maintainability) BEYOND tests-green → severity-gated binary PASS/FAIL. `buildCodeQualityPrompt` (pure renderer, critique-before-verdict) + `codeQualityGatePass` (pure aggregate) + `runCodeQualityJudge` (C-PIN run wrapper on the injected JudgeInvoke seam). Judge-only (EV-051), host-runtime, pinned model + temp 0. **DISTINCT from `code-eval.ts`** (deterministic primitives — no LLM). `DEFAULT_CODE_QUALITY_CRITERIA` + its aggregation rule are FLAGGED-for-operator-sign-off. |
 | `scripts/run-evaluate.ts` | P5 + §9.4 (T1·T3·T4·T5) | hybrid | The real-engine `*evaluate` end-to-end COMPOSER: `tier0Plan` (T1 — code rows decided in the pre-pass, residual judge rows only in packets) → `prepMatrixPackets` (residual-only) → `adaptivePacketPlan` (T5 overload guard, default 1:1) → [dispatch #mode-judge-trajectory] → `aggregateEvaluate` (FAIL-LOUD readiness gate over DISPATCHED ids → `aggregateMatrixScorecard` folding code+judge + T3 `independentVerify` downgrade-only on gating fails → GATE + variance → T4 `consolidateByLocus` lociClusters → EV-051 `routeFailures`→validated HandoverBundle). `maskedScorecard` = the C-PIN byte-identity artifact. |
 | `scripts/render-eval-report.ts` | P4 (D-1/2/3) | code | The v2 reporting (v1 `render-report.ts` untouched): `renderEvalCards` (D-1 terminal per-criterion cards) + `buildEvalReportInput` (derives the rich tabs from real run data — ledger · coverage · cohorts · gating rolls · top-findings · judge-health) + `renderEvalReport` (the operator-APPROVED 5 tabs: ① Overview KPIs+coverage+gating-table+top-findings · ② Trajectory‖Judge ledger + click-row Target-Agent‖Judge side-by-side · ③ Eval Scorecard cohort heatmap + nested subcards + inline calibration · ④ Findings verbatim-evidence + judge chain + agree/revise/refute · ⑤ Self-Eval [INTERNAL, strip-marked]). Consumes the §9.4 judge-walk (`judge_steps[]` + dense map) when present, degrades to the per-trajectory scorecard otherwise. Tabs are `<button>` (no `<s>` strikethrough). Unified design-system tokens (`assets/brand/theme.css`). byte-identical after the `generatedAt` mask. `autoOpenCommand` cross-platform helper (orchestrator fires it). |
 | `scripts/contracts/eval-types.ts` *(extended)* | P2/P2b | code | + `MinedCriterion` (additive subtype of `DiscoveredCriterion`) carrying the §5b `MetricMetadata` (generality·dimension·level·check_method [3-value SF-2 router]·substrate·applies_to·severity·judge_inputs·flag) + the §5c `DiscoveryRationale` (DR-2: targets·why·evidence{grounding observed\|inferred·prevalence k/n·refs}·assumptions·expected_impact). `parseMinedCriterion` enforces the evidence-first gate (OBSERVED ⇒ refs + k>0). |

package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md

@@ -508,6 +508,7 @@ resolution_contract: |
         mode: judge, axis: criterion    => run #mode-judge-criterion (preserves eval-judge);   input is a .task.json.
         mode: verify                    => run #mode-verify (GA-5 result-verifier; reviewer≠judge, downgrade-only); input is a decided verdict + its situation.
         mode: improve                   => run #mode-improve (EDD ③ loop — F19 variance-first then F18 request-amend-reeval, bounded). JUDGE-ONLY: REQUEST, never patch.
+        mode: judge, axis: code-quality => run #mode-judge-code-quality (Wave-2 W2I1 — score a CODE subject's QUALITY for the ⑤ IMPROVE code-target BOTH-gate). JUDGE-ONLY; binary quality PASS/FAIL. Rubric mirrored in scripts/code-quality-verdict.ts.
    4. DISPATCH-INPUT DISAMBIGUATION — the parent hands you files:
         a *.packet.json (MatrixPacket)  => judge/trajectory, ALWAYS (matrix-judge path; no unit field).
         a *.task.json  (JudgeTaskSpec)  => unit.kind: "discover" => discover; else unit.axis: "criterion" => judge/criterion.
@@ -1029,6 +1030,69 @@ never re-implements, the runner. The interface this mode ASSUMES (documented ver
   C-PIN (pinned model + temperature 0).
 - **Additive to the runner:** consumes the assumed eval-runner interface; rebuilds nothing.
 
+## Mode: judge — axis code-quality   ← the ⑤ IMPROVE code-target quality verdict (Wave-2 W2I1)
+
+The **(b) half of the ⑤ IMPROVE loop's code-target BOTH-gate.** For a `code`-kind subject, "converged"
+requires BOTH (a) the subject's OWN test suite green — a hard DETERMINISTIC gate the ai-engineer
+reports and the optimize oracle records as `testsGreen` — AND (b) THIS code-quality verdict passing.
+This mode is (b): score the QUALITY of the applied code change, BEYOND tests-green (a deterministic
+primitive cannot decide spec-faithfulness, scope, error-handling, maintainability — a judge must).
+
+### What it does (per dispatch)
+
+Judge the CODE subject (the amend diff / scaffold against the agentspec definition it must realize)
+on the DRAFT criteria set — **critique-BEFORE-verdict, strictly BINARY** per criterion — and emit a
+severity-gated binary quality **PASS/FAIL**. The authoritative rubric + the drafted default criteria
+live in `scripts/code-quality-verdict.ts` (`DEFAULT_CODE_QUALITY_CRITERIA` — Q1 spec-faithful [crit] ·
+Q2 addresses-diagnosis [high] · Q3 no-regression [high] · Q4 error-handling [high] · Q5 maintainable
+[med]); this mode is that rubric's dispatched form (the provider-callable mirror is the same file,
+mirroring how the other judge prompts mirror their agent modes).
+
+> **⚠️ FLAGGED-FOR-OPERATOR-SIGN-OFF.** The exact acceptance criteria + the severity-gated aggregation
+> rule (CRIT/HIGH `fail`|`uncertain` blocks; MEDIUM advisory) are a DRAFT default, implemented behind
+> a clearly-named constant so the operator can confirm/adjust WITHOUT a loop code change. Do not treat
+> the current set as final.
+
+### Boundaries
+
+- **Judge-only (EV-051):** score the quality; NEVER propose or write a fix (the ⑤ loop's ai-engineer
+  amends). A FAIL just keeps the loop iterating.
+- **Host-runtime, no provider key; C-PIN:** pinned model + temperature 0 — byte-identical reruns.
+- **Separate from tests-green:** you judge quality, NOT whether tests pass; the deterministic test
+  gate is ai-engineer's report, ANDed by the oracle — never fold it into your quality verdict.
+
+## Eval-leg reconcile — criteria MAINTENANCE, not judging (Wave-2 W2I5 · KP-003)
+
+The evaluator owns the **eval leg** of the def → impl → eval triad — the third leg of `#sync-spec`. When
+an impl amends (the ⑤ IMPROVE loop's ai-engineer, or brownfield drift), the eval criteria that ground the
+subject's evaluation can go stale. The reconcile REASONING — which criteria a changed impl makes stale,
+what new criterion it needs — is the **ai-architect's** (`#sync-spec`, session-dispatched, Model-B). The
+evaluator exposes only the deterministic CONTRACT the reconcile reads/writes, in
+`scripts/sync-eval-criteria.ts`:
+
+- `evalLegForSubjectKind(kind)` — the shape resolver: agent/skill/composite → the **eval-suite criteria**
+  (this cell's discovered/maintained criteria); code → the **code-quality criteria**
+  (`DEFAULT_CODE_QUALITY_CRITERIA` / `#mode-judge-code-quality`, Wave-2 W2I1).
+- `flagEvalLegDrift(...)` / `assessEvalLeg(...)` — a pure freshness FLAG (mirrors the builder probe).
+- `reconcileEvalCriteria(request)` — the COMPUTE: upsert the ai-architect-proposed criteria delta by id,
+  append the novel, and NEVER drop a criterion (EV-053, via `assertMonotonicGrowth`). It returns the grown
+  set as DATA (`result.criteria`) — it does **not** touch disk; the reconcile hook COMPUTES, it is not the write.
+- `persistEvalCriteria(result, { cwd, updatedAt? })` (`scripts/persist-eval-criteria.ts`) — the WRITE: the
+  evaluator session PERSISTS `result.criteria` back to the located criteria artifact
+  (`.mutagent/evaluator/living-suite/<leg>.criteria.json`, per-leg — eval-suite for an agent/skill/composite
+  subject, code-quality for a code subject). That write is what BUMPS the artifact's freshness (its mtime — and
+  the optional `updatedAt` marker the builder probe reads), which returns the eval leg to `in-sync`: the loop
+  only closes once the reconciled criteria are on disk, not when they are merely computed.
+
+> **Judge-only PRESERVED (EV-051).** This is criteria MAINTENANCE — growing/revising the criteria SET to
+> match a changed impl — NOT judging a subject. `sync-eval-criteria.ts` scores nothing and decides no
+> pass/fail; it is pure set algebra + a freshness flag (no clock/random/network — C-PIN byte-identical). The
+> `persist-eval-criteria.ts` WRITER is the IO layer (it is the one place that touches disk, mirroring how
+> `artifact-paths.ts` does IO while `living-suite.ts` stays pure); its serialization is still deterministic
+> (stable key order, no clock unless the caller passes `updatedAt`), so a persisted artifact round-trips
+> byte-faithful. The evaluator never mutates the impl and never fixes the subject; it only maintains — and now
+> persists — its own criteria.
+
 ## Boundaries (shared)
 
 judge-only (EV-051) · model-intent-sacred · critique-before-verdict · binary-not-Likert ·

package/.claude/skills/mutagent-evaluator/references/operation-inventory.md

@@ -76,6 +76,7 @@ makes NO pass/fail decision; the LLM/HITL halves are Type B (below).
 | `*build-dataset` expand (EV-046) | `scripts/build-dataset.ts` | `buildCase` · cartesian dimension×value expand · deterministic token-Jaccard near-dup removal · content-derived id · `mergeCases` MONOTONIC merge. The agent PROPOSES realism; the script ENFORCES non-redundancy. Code half of the 3-way `*build-dataset` Hybrid. |
 | `*discover-dataset` distill (EV-047) | `scripts/derive-dataset.ts` | Distill a living regression set from past ✓/✗ — reuses EV-052 `sample-traces.ts` selectors (failure/outlier/uncertainty) + EV-046 monotonic merge; trace→`DatasetCase` whose query IS the real input. Labels already on traces (from `*discover-evals`); makes NO new decision. Type A Code-only. |
 | Living-suite writer (EV-053) | `scripts/living-suite.ts` | `appendOnly` + `assertMonotonicGrowth` — generic-over-`T` append-only writer (a living artifact NEVER shrinks) shared by datasets + criteria. Pure-counter provenance (no clock → byte-identity, C-PIN). Pure set algebra w/ fail-loud monotonicity guard. |
+| Eval-leg reconcile contract (W2I5 · KP-003) | `scripts/sync-eval-criteria.ts` | The eval leg of `#sync-spec` (def → impl → eval triad). `evalLegForSubjectKind` (agent/skill → eval-suite · code → code-quality) · `flagEvalLegDrift`/`assessEvalLeg` (pure freshness flag) · `reconcileEvalCriteria` (upsert-by-id criteria MAINTENANCE via `assertMonotonicGrowth` — never drops, EV-053). Criteria maintenance, NOT judging (EV-051): scores nothing, decides no pass/fail. The reconcile REASONING is `ai-architect #sync-spec`'s (Model-B); this holds only the deterministic contract. Pure (no clock/random/network, C-PIN). |
 | W2 dataset contract | `scripts/contracts/dataset.ts` | TypeBox `Dimension`/`DatasetTuple`/`DatasetCase`/`Dataset` (companion to `schemas/dataset.schema.yaml`) + `tupleKey`. NEW W2-OWN file — does NOT touch shared `contracts/eval-types.ts`. Data contract only. |
 | W2 trust contract | `scripts/contracts/validation.ts` | TypeBox `HumanLabel` (produced by `*review`, consumed by `*validate`) + the confusion-matrix / validation-result shapes. NEW W2-OWN file; disjoint from shared types. Data contract only. |
 

package/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts

@@ -26,7 +26,7 @@ import {
   resolveProfilePaths,
 } from "../load-profile.ts";
 import { loadBundle } from "../load-bundle.ts";
-import { runDeterministic } from "../run-deterministic.ts";
+import { runDeterministic, reexecDeterministicCheck } from "../run-deterministic.ts";
 import { assembleScorecard } from "../assemble-scorecard.ts";
 import { renderReport } from "../render-report.ts";
 import { maskedCanonicalJson } from "../mask.ts";
@@ -107,8 +107,11 @@ export function main(argvIn?: string[]): number {
   // 2. Run-bundle
   const { bundle } = loadBundle(args.bundleDir, args.runId);
 
-  // 3. Deterministic pass
-  const det = runDeterministic(matrix, bundle);
+  // 3. Deterministic pass — WIRE the real live re-exec (Item #4): every mapped
+  // deterministic row runs an actual checkMethod-keyed check over the produced
+  // artifact instead of a presence proxy. In-cell, no sub-agents (PR-ORCH-01),
+  // judge-only (EV-051), C-PIN byte-stable.
+  const det = runDeterministic(matrix, bundle, { liveReexec: reexecDeterministicCheck });
 
   // 4. Two-track scorecard
   const scorecard = assembleScorecard({

package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts

@@ -0,0 +1,285 @@
+/**
+ * scripts/code-quality-verdict.ts — the CODE-QUALITY judge mode (Wave-2 W2I1).
+ * ---------------------------------------------------------------------------
+ * A JUDGE mode that scores a CODE subject's QUALITY (the amended/scaffolded
+ * implementation of a `code`-kind subject) and emits a binary PASS/FAIL quality
+ * verdict with a critique-before-verdict per criterion. It is the (b) half of the
+ * ⑤ IMPROVE loop's code-target BOTH-gate: "converged" for a `code` subject = the
+ * subject's OWN test suite green (a deterministic gate the session records as
+ * `testsGreen`) AND this code-quality verdict PASSes. Neither alone converges.
+ *
+ * DISTINCT FROM `code-eval.ts`. That file is the DETERMINISTIC code-track primitive
+ * library (presence/string-equality/... over an EvalTrace — no LLM, byte-identical).
+ * This file is an LLM JUDGE mode: it scores the irreducibly-subjective quality of a
+ * code implementation (spec-faithfulness, maintainability, error-handling, ...) that
+ * a deterministic primitive cannot decide.
+ *
+ * INVARIANTS (consistent with the existing evaluator cell):
+ *   - JUDGE-ONLY (EV-051). It scores; it NEVER fixes. Failures route to the ⑤ loop's
+ *     ai-engineer amend (via the loop, not from here).
+ *   - HOST-RUNTIME. Under the DEFAULT agent-dispatch substrate the authoritative
+ *     rubric lives in the evaluator agent def (`#mode-judge-code-quality`), and the
+ *     verdict comes from a dispatched subagent — this file's prompt is the
+ *     provider-callable MIRROR for the OPTIONAL in-house / export substrate (same
+ *     operator-named exception as judge-prompt-template.ts). No provider key on the
+ *     default path.
+ *   - PINNED model + temperature 0 (C-PIN). `runCodeQualityJudge` THROWS if the pin
+ *     is not (modelId present AND temperature === 0) — reruns must be byte-identical.
+ *   - PURE renderer + a thin injected-seam run wrapper. No clock, no random, no
+ *     network; the LLM call is the injected `JudgeInvoke` seam so tests drive it.
+ */
+import { parseCritiqueVerdict, type JudgeInvoke } from "./determine-outcome.ts";
+import type { JudgePin } from "./build-evals.ts";
+import type { CritiqueVerdict } from "./contracts/eval-types.ts";
+
+/** Severity of a code-quality criterion — gates the aggregate (CRIT/HIGH block). */
+export const CodeQualitySeverity = {
+  Critical: "critical",
+  High: "high",
+  Medium: "medium",
+} as const;
+export type CodeQualitySeverityValue =
+  (typeof CodeQualitySeverity)[keyof typeof CodeQualitySeverity];
+
+/** One binary code-quality criterion the judge scores. */
+export interface CodeQualityCriterion {
+  /** Stable id (byte-identity key + report anchor). */
+  id: string;
+  /** The one-line binary statement the judge decides pass/fail on. */
+  statement: string;
+  /** What a PASS looks like (concrete, so the judge is not guessing). */
+  passDefinition: string;
+  /** What a FAIL looks like. */
+  failDefinition: string;
+  /** Gating weight — a `fail`/`uncertain` on a CRIT/HIGH criterion blocks the gate. */
+  severity: CodeQualitySeverityValue;
+}
+
+/**
+ * ⚠️ FLAGGED-FOR-OPERATOR-SIGN-OFF (Wave-2 W2I1). This is the DRAFT default set of
+ * acceptance criteria that define "code quality-pass" BEYOND tests-green. It is
+ * implemented behind this clearly-named default so the operator can confirm/adjust
+ * the criteria + their severities WITHOUT a code change to the loop. The picks
+ * below are a conservative, subject-agnostic starting point — NOT a final contract.
+ *
+ * The five drafted criteria (each BINARY, critique-before-verdict):
+ *   Q1 spec-faithful  (critical) — the impl realizes the agentspec DEFINITION the
+ *                                  amend targeted; no dropped tool / JTBD / silently
+ *                                  altered system_prompt (def→impl cascade held).
+ *   Q2 addresses-diagnosis (high) — the change actually targets the diagnosed
+ *                                  root-cause locus, not an adjacent symptom.
+ *   Q3 no-regression  (high)    — no unrelated behavior removed/broken; scope stayed
+ *                                  on what the remedy named (scope discipline).
+ *   Q4 error-handling (high)    — failure paths are handled, not silently dropped
+ *                                  (recovery/observability preserved).
+ *   Q5 maintainable   (medium)  — readable, framework-idiomatic, no dead code /
+ *                                  copy-paste / obvious footgun introduced.
+ *
+ * Aggregation (also FLAGGED): the gate is severity-gated — a `fail` OR `uncertain`
+ * on any CRITICAL/HIGH criterion blocks the quality-pass; MEDIUM is advisory. The
+ * operator may instead choose strict-all-pass or a different severity map.
+ */
+export const DEFAULT_CODE_QUALITY_CRITERIA: readonly CodeQualityCriterion[] = [
+  {
+    id: "Q1-spec-faithful",
+    statement: "The implementation realizes the agentspec definition the amend targeted.",
+    passDefinition:
+      "PASS: every spec-declared tool / job-to-be-done is present and the operative system_prompt is unchanged except where the amend intended; the def→impl cascade held.",
+    failDefinition:
+      "FAIL: a spec-declared tool or JTBD is missing, or the system_prompt was altered beyond the amend intent (the impl forked from the SSoT).",
+    severity: CodeQualitySeverity.Critical,
+  },
+  {
+    id: "Q2-addresses-diagnosis",
+    statement: "The change targets the diagnosed root-cause locus, not an adjacent symptom.",
+    passDefinition:
+      "PASS: the edit is at (or directly causes a fix at) the RCA-named locus / the failing-case root.",
+    failDefinition:
+      "FAIL: the change edits an unrelated place or patches a symptom while the root cause remains.",
+    severity: CodeQualitySeverity.High,
+  },
+  {
+    id: "Q3-no-regression",
+    statement: "The change introduces no unrelated regression and stays in scope.",
+    passDefinition:
+      "PASS: only the remedy-named surface changed; no unrelated behavior was removed or broken.",
+    failDefinition:
+      "FAIL: unrelated code was changed/removed, or a previously-working path was broken.",
+    severity: CodeQualitySeverity.High,
+  },
+  {
+    id: "Q4-error-handling",
+    statement: "Failure paths are handled, not silently dropped.",
+    passDefinition:
+      "PASS: error/failure branches surface, recover, or log per the subject's contract; no new silent drop.",
+    failDefinition:
+      "FAIL: a failure is swallowed with no recovery/notify/observability (a silent-drop regression).",
+    severity: CodeQualitySeverity.High,
+  },
+  {
+    id: "Q5-maintainable",
+    statement: "The code is readable, framework-idiomatic, and free of obvious footguns.",
+    passDefinition:
+      "PASS: clear structure, idiomatic to the target framework, no dead code / copy-paste / obvious footgun introduced.",
+    failDefinition:
+      "FAIL: unreadable/duplicated/dead code or an obvious maintainability footgun was introduced.",
+    severity: CodeQualitySeverity.Medium,
+  },
+] as const;
+
+/** The code subject under quality review (minimal, subject-agnostic). */
+export interface CodeQualitySubject {
+  /** Subject id (spec_id / target name) — provenance only. */
+  subjectId: string;
+  /** The target framework/harness the impl was built into (context for the judge). */
+  framework: string;
+  /** A compact summary of the agentspec DEFINITION the impl must realize (the SSoT). */
+  specSummary: string;
+  /** The applied change under review — the amend diff, or the scaffold summary. */
+  appliedChange: string;
+  /**
+   * The diagnosed root-cause / remedy locus this change was meant to address (feeds
+   * Q2). Empty for a fresh scaffold with no prior diagnosis.
+   */
+  diagnosisLocus?: string;
+}
+
+/** One criterion's judged result (critique-before-verdict, binary). */
+export interface CodeQualityCriterionVerdict {
+  criterionId: string;
+  severity: CodeQualitySeverityValue;
+  result: CritiqueVerdict["result"]; // pass | fail | uncertain
+  critique: string;
+  confidence: number;
+}
+
+/** The aggregated code-quality verdict for one subject. */
+export interface CodeQualityVerdict {
+  subjectId: string;
+  /** The binary quality gate — the (b) half of the ⑤ code-target BOTH-gate. */
+  pass: boolean;
+  perCriterion: CodeQualityCriterionVerdict[];
+  /** C-PIN provenance stamped on the verdict (byte-identity guarantee). */
+  pinned: JudgePin;
+}
+
+/**
+ * Render the code-quality judge prompt for ONE criterion — the provider-callable
+ * mirror of the `#mode-judge-code-quality` rubric. Critique-BEFORE-verdict, strictly
+ * BINARY (no Likert), judge-only. No decision is made here.
+ */
+export function buildCodeQualityPrompt(
+  criterion: CodeQualityCriterion,
+  subject: CodeQualitySubject,
+): { system: string; user: string } {
+  const system = [
+    "You are a BINARY Pass/Fail code-QUALITY judge for ONE criterion of a CODE",
+    "subject (an agent/tooling implementation). Judge exactly this criterion and",
+    "nothing else. You are a JUDGE ONLY — you NEVER propose or write a fix.",
+    "",
+    "Quality is assessed BEYOND the subject's test suite (tests-green is a separate",
+    "deterministic gate). Judge the code as written against the criterion.",
+    "",
+    `Criterion (${criterion.id}, severity ${criterion.severity}): ${criterion.statement}`,
+    criterion.passDefinition,
+    criterion.failDefinition,
+    "",
+    "Outcomes are strictly BINARY: pass or fail (use uncertain ONLY when the change",
+    "genuinely lacks the evidence to decide). NO Likert scales, NO 1-5 / letter",
+    "grades, NO partial credit.",
+    "",
+    "Output STRICT JSON with the critique BEFORE the verdict (reason first, then",
+    "commit):",
+    '{ "critique": "<evidence-citing assessment>", "result": "pass"|"fail"|"uncertain",',
+    '  "confidence": <0..1> }',
+  ].join("\n");
+
+  const user = [
+    `Subject: ${subject.subjectId}  ·  target framework: ${subject.framework}`,
+    "",
+    "Agentspec definition the implementation must realize (the SSoT):",
+    subject.specSummary,
+    "",
+    subject.diagnosisLocus !== undefined && subject.diagnosisLocus.length > 0
+      ? `Diagnosed root-cause / remedy locus (for the addresses-diagnosis check):\n${subject.diagnosisLocus}\n`
+      : "",
+    "Applied change under review (diff or scaffold summary):",
+    subject.appliedChange,
+  ]
+    .filter((l) => l !== "")
+    .join("\n");
+
+  return { system, user };
+}
+
+/**
+ * The PURE aggregation: severity-gate the per-criterion verdicts into ONE binary
+ * quality-pass. FLAGGED default rule — a `fail` OR `uncertain` on any CRITICAL/HIGH
+ * criterion blocks the pass; MEDIUM is advisory (never blocks). Deterministic.
+ */
+export function codeQualityGatePass(
+  perCriterion: readonly CodeQualityCriterionVerdict[],
+): boolean {
+  for (const v of perCriterion) {
+    const gating =
+      v.severity === CodeQualitySeverity.Critical || v.severity === CodeQualitySeverity.High;
+    if (gating && v.result !== "pass") return false;
+  }
+  return true;
+}
+
+/**
+ * Run the code-quality judge over the criteria set under a PINNED model. THROWS if
+ * the pin is not (modelId present AND temperature === 0) — C-PIN (byte-identical
+ * reruns). Renders each criterion → the injected `JudgeInvoke` seam → parses
+ * (critique-before-verdict) → severity-gated aggregate. Under the DEFAULT
+ * agent-dispatch substrate the verdicts instead come from a dispatched
+ * `#mode-judge-code-quality` subagent (verdict files) and this wrapper is not used.
+ * Deterministic given (subject, judge, pin, criteria).
+ */
+export async function runCodeQualityJudge(
+  subject: CodeQualitySubject,
+  judge: JudgeInvoke,
+  pin: JudgePin,
+  criteria: readonly CodeQualityCriterion[] = DEFAULT_CODE_QUALITY_CRITERIA,
+): Promise<CodeQualityVerdict> {
+  if (typeof pin.modelId !== "string" || pin.modelId.length === 0) {
+    throw new Error(
+      "runCodeQualityJudge: judge is not pinned (missing modelId). MODEL INTENT IS " +
+        "SACRED — a non-pinned judge can never produce a verdict (C-PIN).",
+    );
+  }
+  if (pin.temperature !== 0) {
+    throw new Error(
+      `runCodeQualityJudge: judge temperature=${pin.temperature} (!= 0) — not pinned. ` +
+        "MODEL INTENT IS SACRED: reruns must be byte-identical (C-PIN); refusing.",
+    );
+  }
+  if (criteria.length === 0) {
+    throw new Error(
+      "runCodeQualityJudge: empty criteria set — a code-quality verdict with no " +
+        "criteria would vacuously pass (a false-green). Refusing.",
+    );
+  }
+
+  const perCriterion: CodeQualityCriterionVerdict[] = [];
+  for (const criterion of criteria) {
+    const { system, user } = buildCodeQualityPrompt(criterion, subject);
+    const raw = await judge(system, user);
+    const v = parseCritiqueVerdict(raw);
+    perCriterion.push({
+      criterionId: criterion.id,
+      severity: criterion.severity,
+      result: v.result,
+      critique: v.critique,
+      confidence: v.confidence,
+    });
+  }
+
+  return {
+    subjectId: subject.subjectId,
+    pass: codeQualityGatePass(perCriterion),
+    perCriterion,
+    pinned: { modelId: pin.modelId, temperature: pin.temperature },
+  };
+}

package/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts

@@ -0,0 +1,232 @@
+/**
+ * scripts/persist-eval-criteria.ts — the EVAL-LEG on-disk WRITER (Wave-2 FU57 · KP-003).
+ * ---------------------------------------------------------------------------
+ * The IO half of the eval-leg reconcile. `sync-eval-criteria.ts` COMPUTES the grown
+ * criteria set as DATA (`reconcileEvalCriteria` — pure upsert-by-id, monotonic, never
+ * drops); it never touches disk. This module is the deterministic WRITER the evaluator
+ * SESSION calls to PERSIST that computed set back to the located eval-suite criteria
+ * artifact — the write that bumps the artifact's freshness and thereby returns the eval
+ * leg of `#sync-spec` to `in-sync` (the builder freshness probe reads the artifact's
+ * mtime / `updatedAt`; a fresh write makes the eval criteria as new as the amended impl).
+ *
+ * SEPARATION OF CONCERNS (mirrors `artifact-paths.ts` + `aggregate-discover.ts` doing IO
+ * while `living-suite.ts` stays pure): the reconcile COMPUTE is pure/C-PIN; this WRITE is
+ * the IO layer. The SERIALIZATION here is still deterministic (stable key order, no clock
+ * embedded unless the caller passes an explicit `updatedAt`), so a persisted artifact
+ * round-trips byte-faithful.
+ *
+ * JUDGE-ONLY PRESERVED (EV-051): this file writes CRITERIA (the evaluator maintaining its
+ * own eval-suite / code-quality criteria set) — it NEVER scores a subject, NEVER decides a
+ * pass/fail, and NEVER dispatches an agent (Model-B: code is a deterministic writer; the
+ * reconcile REASONING is the ai-architect's, session-dispatched).
+ *
+ * LOCATION + FORMAT (mirror the living-suite persistence; FLAGGED-for-operator, see
+ * `EVAL_CRITERIA_ARTIFACT_FILENAMES`): the artifact lands under the evaluator's namespaced
+ * `.mutagent/evaluator/living-suite/` root (`artifact-paths.ts` `livingSuiteDir`, auto-
+ * gitignored, no-spillover-guarded) as one JSON file per LEG — the same dir + extension the
+ * builder `check-sync-spec.ts` `locateEvalCriteria` auto-discovers.
+ */
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+import { assertUnderRoot, livingSuiteDir } from "./artifact-paths.ts";
+import {
+  EvalCriterionSchema,
+  EvalLegKind,
+  reconcileEvalCriteria,
+  type EvalCriteriaReconcileRequest,
+  type EvalCriteriaReconcileResult,
+  type EvalCriterion,
+  type EvalLegKindValue,
+} from "./sync-eval-criteria.ts";
+
+// ── the on-disk artifact shape (mirror LivingSuite `{ …, provenance }`) ───────
+
+/**
+ * The persisted eval-criteria artifact — one per LEG. Mirrors the living-suite shape
+ * (a keyed collection + provenance), specialized to the reconcile result: the maintained
+ * `criteria` set, the reconcile `provenance` counters, and an OPTIONAL `updatedAt`
+ * freshness marker. `updatedAt` is the ONE clock-bearing field and it is NEVER generated
+ * here — the caller (the session, Model-B) passes it when it wants a content-level
+ * freshness stamp the builder probe's `parseEvalUpdatedAt` can read; omitted, freshness
+ * rides on the file's mtime alone (write bumps it). `additionalProperties: false` keeps the
+ * header tight while each criterion stays forward-parsing (EvalCriterionSchema is open).
+ */
+export const EvalCriteriaArtifactSchema = Type.Object(
+  {
+    subjectId: Type.String({ minLength: 1 }),
+    leg: Type.Union([
+      Type.Literal(EvalLegKind.EvalSuite),
+      Type.Literal(EvalLegKind.CodeQuality),
+    ]),
+    criteria: Type.Array(EvalCriterionSchema),
+    provenance: Type.Object(
+      { added: Type.Number(), updated: Type.Number(), total: Type.Number() },
+      { additionalProperties: false },
+    ),
+    updatedAt: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+);
+export type EvalCriteriaArtifact = Static<typeof EvalCriteriaArtifactSchema>;
+
+/**
+ * FLAGGED-for-operator: the per-leg artifact FILENAME under `.mutagent/evaluator/living-suite/`.
+ * The sensible default that mirrors the living-suite location AND the builder probe's
+ * conventional fallback names (`eval-criteria` / `code-quality-criteria`), one file per leg
+ * so the two legs never collide. Both are `.json` under the dir `check-sync-spec.ts`
+ * `locateEvalCriteria` walks (`*.{yaml,yml,json}`), so a written artifact is auto-discovered
+ * as the eval-leg freshness anchor with no extra wiring. (A workspace evaluates ONE subject,
+ * so only its leg's file exists; multi-subject-per-workspace would want a `<subjectId>/`
+ * sub-dir — deferred, flagged.)
+ */
+export const EVAL_CRITERIA_ARTIFACT_FILENAMES: Readonly<Record<EvalLegKindValue, string>> = {
+  [EvalLegKind.EvalSuite]: "eval-suite.criteria.json",
+  [EvalLegKind.CodeQuality]: "code-quality.criteria.json",
+};
+
+/**
+ * Resolve the located eval-criteria artifact path for a leg:
+ * `<cwd>/.mutagent/evaluator/living-suite/<leg>.criteria.json`. Routes through the
+ * `artifact-paths.ts` no-spillover guard (`assertUnderRoot`), so the writer can NEVER
+ * escape the evaluator's namespaced root. `cwd` passed in (pure path construction).
+ */
+export function evalCriteriaArtifactPath(leg: EvalLegKindValue, cwd?: string): string {
+  return assertUnderRoot(join(livingSuiteDir(cwd), EVAL_CRITERIA_ARTIFACT_FILENAMES[leg]), cwd);
+}
+
+// ── serialize / build (pure, deterministic — no clock) ───────────────────────
+
+/** Build the on-disk artifact from a reconcile RESULT. Pure. `updatedAt` passed in. */
+export function evalCriteriaArtifactFromResult(
+  result: EvalCriteriaReconcileResult,
+  updatedAt?: string,
+): EvalCriteriaArtifact {
+  return {
+    subjectId: result.subjectId,
+    leg: result.leg,
+    criteria: result.criteria,
+    provenance: result.provenance,
+    ...(updatedAt !== undefined ? { updatedAt } : {}),
+  };
+}
+
+/**
+ * Serialize an artifact to its on-disk JSON — DETERMINISTIC: explicit key order, 2-space
+ * indent, trailing newline. Same artifact → byte-identical string (round-trip faithful);
+ * no clock unless the artifact already carries a caller-supplied `updatedAt`.
+ */
+export function serializeEvalCriteriaArtifact(artifact: EvalCriteriaArtifact): string {
+  const ordered: EvalCriteriaArtifact = {
+    subjectId: artifact.subjectId,
+    leg: artifact.leg,
+    criteria: artifact.criteria,
+    provenance: artifact.provenance,
+    ...(artifact.updatedAt !== undefined ? { updatedAt: artifact.updatedAt } : {}),
+  };
+  return `${JSON.stringify(ordered, null, 2)}\n`;
+}
+
+/** Guarded parse of a persisted artifact. THROWS with the first schema error. PURE. */
+export function parseEvalCriteriaArtifact(value: unknown): EvalCriteriaArtifact {
+  if (!Value.Check(EvalCriteriaArtifactSchema, value)) {
+    const first = [...Value.Errors(EvalCriteriaArtifactSchema, value)][0];
+    throw new Error(
+      `parseEvalCriteriaArtifact: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid eval-criteria artifact"}`,
+    );
+  }
+  return value;
+}
+
+// ── the WRITER + reader (the IO layer) ───────────────────────────────────────
+
+/** The result of a persist: WHERE it landed + WHAT was written (artifact + bytes). */
+export interface PersistEvalCriteriaResult {
+  /** the located artifact path the criteria were written to. */
+  path: string;
+  /** the artifact that was serialized. */
+  artifact: EvalCriteriaArtifact;
+  /** the exact bytes written (for a byte-faithful round-trip assertion). */
+  json: string;
+}
+
+/**
+ * PERSIST a reconcile result's `criteria` back to the located eval-suite criteria artifact
+ * — the WRITE that returns the eval leg to `in-sync`. Resolves the per-leg path under
+ * `.mutagent/evaluator/living-suite/` (no-spillover-guarded via `artifact-paths.ts`),
+ * ensures the dir, and writes the deterministic JSON. Handles BOTH legs by `result.leg`:
+ * the eval-suite leg (agent/skill/composite subject) and the code-quality leg (a code
+ * subject, incl. an operator-overridden code-quality set). The freshness bump is the file
+ * write itself (mtime advances → the builder probe reads the eval leg as fresh); pass
+ * `options.updatedAt` (ISO 8601) to ALSO stamp a content-level freshness marker the probe's
+ * `parseEvalUpdatedAt` reads. IO — NOT pure (that is its job); serialization is deterministic.
+ */
+export function persistEvalCriteria(
+  result: EvalCriteriaReconcileResult,
+  options: { cwd?: string; updatedAt?: string } = {},
+): PersistEvalCriteriaResult {
+  const artifact = evalCriteriaArtifactFromResult(result, options.updatedAt);
+  const path = evalCriteriaArtifactPath(result.leg, options.cwd);
+  const json = serializeEvalCriteriaArtifact(artifact);
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, json);
+  return { path, artifact, json };
+}
+
+/** Read + validate a persisted artifact (round-trip). THROWS on a schema violation. */
+export function readEvalCriteriaArtifact(path: string): EvalCriteriaArtifact {
+  return parseEvalCriteriaArtifact(JSON.parse(readFileSync(path, "utf8")) as unknown);
+}
+
+/**
+ * Load the CURRENTLY-persisted criteria for a leg (the `existing` set the next reconcile
+ * grows). Returns `[]` when no artifact exists yet (the cold / first-reconcile case) so a
+ * caller can feed it straight into an `EvalCriteriaReconcileRequest.existing`. Reads only.
+ */
+export function loadExistingEvalCriteria(leg: EvalLegKindValue, cwd?: string): EvalCriterion[] {
+  const path = evalCriteriaArtifactPath(leg, cwd);
+  if (!existsSync(path)) return [];
+  return readEvalCriteriaArtifact(path).criteria;
+}
+
+/**
+ * The eval-criteria artifact's effective freshness epoch (seconds) — the evaluator-side
+ * MIRROR of the builder `check-sync-spec.ts` eval-leg compare: `max(updatedAt, mtime)`
+ * (git omitted — kept in-package + git-independent). `null` when the artifact is absent.
+ * Feed this as `evalCriteriaEpoch` to `flagEvalLegDrift` to confirm a fresh write closed
+ * the loop (`in-sync`). Reads mtime (IO).
+ */
+export function evalCriteriaEffectiveEpoch(path: string): number | null {
+  if (!existsSync(path)) return null;
+  const epochs: number[] = [];
+  try {
+    const parsed = readEvalCriteriaArtifact(path);
+    if (parsed.updatedAt !== undefined) {
+      const ms = Date.parse(parsed.updatedAt);
+      if (Number.isFinite(ms)) epochs.push(Math.floor(ms / 1000));
+    }
+  } catch {
+    // a corrupt artifact still has an mtime — fall back to it for freshness.
+  }
+  epochs.push(Math.floor(statSync(path).mtimeMs / 1000));
+  return Math.max(...epochs);
+}
+
+/**
+ * Convenience for the session: COMPUTE the reconcile (pure) THEN PERSIST it (IO) in one
+ * call — the exact "reconcile → write → in-sync" step the `#sync-spec` eval leg performs.
+ * Still Model-B: the criteria DELTA (`request.proposed`) is the ai-architect's reasoning;
+ * this only applies it deterministically and writes it. Returns the compute result + the
+ * persist result together.
+ */
+export function reconcileAndPersistEvalCriteria(
+  request: EvalCriteriaReconcileRequest,
+  options: { cwd?: string; updatedAt?: string } = {},
+): { result: EvalCriteriaReconcileResult; persisted: PersistEvalCriteriaResult } {
+  const result = reconcileEvalCriteria(request);
+  const persisted = persistEvalCriteria(result, options);
+  return { result, persisted };
+}