@mutagent/evaluator 0.1.0-alpha.6 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/golden/judge-trajectory.prose.md

@@ -0,0 +1,69 @@
+# golden/judge-trajectory.prose.md — the C-PIN prose surface (VERSIONED anchor)
+
+> The #mode-judge-trajectory judge constructs its prompt AT REASON-TIME from the
+> packet + this mode's prose — so THIS prose is the load-bearing C-PIN surface.
+> Any change to the workflow below is a judge-behavior change and MUST bump the
+> version here (the W-gate byte-identity proof anchors against this snapshot).
+
+## v3.1 — 2026-07-24 · REF-CONTRACT HARDENING (the first-real-run verify-surface flip)
+
+Change vs v3.0 (ONE workflow line — GATHER): the ref contract is made explicit
+after the first real run showed judge-cited packet-shaped refs reading as DEAD
+at deterministic re-verify (both verifier-UPHELD HIGH fails silently downgraded
+→ gate flipped fail→incomplete). Now: `ref.obs` = the packet's `trajectoryId`
+(NEVER an observation name — batch-order first-match can locate the wrong
+trajectory), `ref.path` = packet-shaped `trajectory.N`/`transcript.N` OR
+trace-shaped `observations.N` (both re-resolve — the verifier reconstructs the
+packet views via `verifySituationViews`), dot form canonical, brackets
+tolerated (`readPath` normalizes). VERBATIM snapshot of the mode workflow
+follows.
+
+```yaml
+  - evaluate:
+      meta:
+        what: "Score ONE trajectory against the WHOLE eval matrix (was the eval-matrix-judge agent). The DEFAULT/headline judging cell."
+        does: "Reads a MatrixPacket (the whole matrix + one trajectory + transcript + pinned envelope), BUILDS its judging prompt from the packet + the shared write-judge-prompt.md, scores every criterion for that trajectory, writes a per-trajectory MatrixVerdictFile."
+        why:  "Per-TRAJECTORY fan-out (one judge scores the whole matrix for one session) = high throughput across many sessions. EV-048. This is the headline *evaluate path."
+        how:  "Unlike the criterion axis, NO script renders the prompt — the parent's matrix-judge.ts PREPs only a DATA packet (keyed by trajectoryKey); THIS mode constructs the prompt at reason-time. Its prompt-construction prose is therefore the load-bearing C-PIN surface (golden/judge-trajectory.prose.md)."
+      display: "Score one trajectory against the WHOLE eval matrix (headline)"
+      description: |
+        PURPOSE: the DEFAULT headline judging cell — one judge per trajectory scores
+        the entire eval matrix for that session → per-criterion verdicts. Critique-
+        before-verdict, binary, inaction-can-be-success, whole-matrix-per-trajectory.
+
+        USAGE: dispatched by the parent, MASS-PARALLEL (one per trajectory). Reads
+        <trajectory_key>.packet.json (a MatrixPacket — DATA, NOT a rendered prompt);
+        BUILDS the prompt from it + write-judge-prompt.md; writes the MatrixVerdictFile.
+      dispatch: { mode: judge, axis: trajectory }
+      pre_gate.loads:
+        - "references/write-judge-prompt.md"    # SHARED 4-component contract (both judge axes; now BIND-before-judge)
+        - "references/grounded-adjudication.md"  # GA doctrine: bind · gather refs · typed assumptions · abstain · verify
+        - "references/workflows/orchestrator-protocol.md"  # Step 2 dispatch FSM
+      workflow:
+        - "Pre-read references/write-judge-prompt.md (the 4-component judging contract — your lens) + references/grounded-adjudication.md"
+        - "Read your assigned <trajectory_key>.packet.json (MatrixPacket: subject · trajectoryId · WHOLE matrix · trajectory · transcript · pinned envelope) — judge exactly this, never re-derive the data"
+        - "Frame the trajectory in its ROUTE/intended-outcome (CONTEXT) before scoring"
+        - "v3 LAYERED WALK (D1=B+, operator-frozen): the walk IS a fixed layer sequence you run INSIDE this one dispatch. Honor `packet.layerScope` when present (absent/empty = FULL walk). The phases: (L0 code checks) if the code-eval library is present at scripts/code-eval-library.ts, RUN it yourself via Bash over the packet's trajectory JSON and record each hit as codeEvalHits[{pattern,anchor,detail}] — a fired pattern is evidence, not a verdict; (L1 outcome) classify the SCENARIO from the inbound intent + tool inventory, derive its expectedExit[] terminal actions, glance ONLY at end-state evidence and emit layerVerdicts[{layer:'L1',verdict,scenario,expectedExit,refs,note}]; (STOPPING RULES) if L1 passes AND every criterion is answerable from evidence gathered so far → you may set earlyExit:'L1' and mark L2-L4 as {verdict:'skipped'} — but a criterion may NEVER go unverdicted because the agent did the right thing; if ANY criterion is unanswerable OR L1 fails/undecidable → descend; (L2 trajectory) actual-vs-expected decision path, read from the END, emit layerVerdicts L2 (+ divergence in localize); (L3 tool outputs) per-observation structural validity + completeness — external-data correctness is UNVERIFIABLE (fence), a failed tool makes everything downstream suspect; emit layerVerdicts L3 (+firstProblemOrigin); (L4 context) for each critical operational send: was required context retrieved AND present in history AT THAT TIME (chronology-sorted!) — distinguish gapKind retrieval (never fetched) vs attention (present, ignored); emit layerVerdicts L4. Record layersEngaged. CONFLICT DISCIPLINE (OT-1): when your own layers disagree (e.g. L1 pass vs L2 fail) emit BOTH verdicts honestly with a note — NEVER reconcile them yourself; the aggregate surfaces the conflict for a human."
+        - "Score EVERY criterion in the matrix for THIS trajectory: read only what the row needs (judgeInputs); compare against statement + passCondition — criteria are the SUPER-LAYER: they may bind evidence from ANY walk phase"
+        - "BIND (L1) per row: each criterion TERM must resolve in THIS trajectory; an unbound term ⇒ uncertain + blockedBy:{kind:factual-intent} (INDETERMINATE), ABSTAIN — never fail"
+        - "GATHER: cite a structured ref {obs,path,value} for the claim AND any absence (ref.obs = the packet's trajectoryId, NEVER an observation name; ref.path = packet-shaped trajectory.N/transcript.N OR trace-shaped observations.N — dot form canonical, brackets tolerated); surface TYPED assumptions for any ungroundable premise → uncertain(blockedBy)"
+        - "Critique BEFORE verdict; commit to result ∈ {pass,fail} (uncertain when the INPUTS can't decide — abstain-on-silence); binary, severity is the row's"
+        - "Inaction-can-be-success: a correct HOLD (no send during a non-critical outbound_guard) is a PASS even with zero tool calls"
+        - "Write <trajectory_key>.verdict.json (a MatrixVerdictFile {trajectoryId, judgeModel, temperature:0, verdicts[]} — each verdict may carry refs?/assumptions?/blockedBy?)"
+        - "EMIT-CONTRACT (HARD, WS-1): on every COMPLETE-fidelity trajectory you MUST also persist the §9.4 walk — `understanding` (M2), `expectedTrajectory` (M3), `agentSteps` (the target step lane), and `judgeSteps` (your anchored reasoning). These are NOT optional: the parent runs a machine `assessEmitCompleteness` gate (scripts/emit-completeness.ts) that LOUDLY flags any wholly-dropped field, and a verdict missing them STARVES the report's Trajectory (§2) + Self-Eval (§5) tabs. `agentSteps` is factual trace data (reconstruct it from the ordered tool steps you were given — order by startTime, the observations array is reverse-chronological); M2/M3/judgeSteps are YOUR reasoning. Only an INCOMPLETE (node-1) trajectory is exempt (it emits verdicts:[] and no walk)."
+        - "v3 EMIT-CONTRACT ADDITIONS (E-NEW): alongside the §9.4 walk, persist layerVerdicts[] (one per engaged layer, skipped ones marked), earlyExit?, layersEngaged, codeEvalHits[] (when the library ran), and the emissions SELF-MANIFEST {emitted[], missing[{key,reason}]} — list honestly what you gathered vs what this contract expects; a missing emission with a stated reason is a NAMED degrade, a silent drop is a defect (MR-5 catches it same-run)."
+        - "Verdicts are independently VERIFIED (#mode-verify · ≠ judge · downgrade-only); a CRIT/HIGH uncertain rolls the run up to INCOMPLETE at the gate (no false-green)"
+      compresses:
+        - "read MatrixPacket DATA; frame in ROUTE (CONTEXT)"
+        - "build judging prompt from packet + write-judge-prompt.md"
+        - "BIND terms · GATHER refs + typed assumptions · abstain on silence"
+        - "score whole matrix per trajectory, critique-before-verdict"
+        - "emit MatrixVerdictFile (then independent VERIFY, downgrade-only)"
+      preserves: "the eval-matrix-judge discipline (the headline cell; prompt-construction prose at golden/judge-trajectory.prose.md) — folded INLINE as 'Mode: judge — axis trajectory' (below); the standalone assets/agents/eval-matrix-judge.md was RETIRED in the 5→3 consolidation (Phase 3a, df6a6e8c8). This file is its canonical home."
+```
+
+## history
+
+- v3.1 (2026-07-24): GATHER ref-contract hardening — obs=trajectoryId, packet-OR-trace-shaped paths, dot canonical/brackets tolerated (fixes the verify-surface fail→incomplete flip found on the first real-trace run).
+- v3.0 (2026-07-24): layered walk + v3 emit-contract (evaluator-v3, operator-frozen).
+- v2.2 and earlier: prose lived inline in evaluator.md only; no on-disk snapshot existed (this file materializes the long-referenced golden anchor).

package/.claude/skills/mutagent-evaluator/references/data-registry.md

@@ -0,0 +1,43 @@
+# The data registry — every entity this skill can produce or consume
+
+> The agent's general knowledge of available entities (the resilience mechanism from
+> the data-availability audit §⓪). Purpose: so that when an expected entity is ABSENT,
+> the absence is **NAMED in words** — never a silent blank, never a fabricated value.
+> A renderer that cannot source a field says so ("not recorded by this run"); a judge
+> that cannot gather an emission lists it in its `emissions.missing[]` self-manifest.
+
+## Run artifacts (the `.mutagent/evaluator/runs/<runId>/` dot-root, gitignored)
+
+| entity | produced by | consumed by | absence → |
+|---|---|---|---|
+| `run-input.full.json` | run PREP (`run-evaluate`) | AGGREGATE, the report renderers | a run cannot aggregate — fail loud |
+| `packets/<trajectoryKey>.packet.json` | `prepMatrixPackets` | one dispatched judge each | the readiness gate throws (no silent partial roll-up) |
+| `verdicts/<trajectoryKey>.verdict.json` | a dispatched `#mode-judge-trajectory` agent | AGGREGATE | same — every judged trajectory MUST have one |
+| `verdicts/*.verify.json` | a dispatched `#mode-verify` reviewer | the T3 independent-verify ledger | self-verify fallback (a NAMED degrade, not a silent downgrade) |
+| `near-duplicate-decisions.json` | `aggregateDiscover` | the next run's brief (no re-proposal) | first run — key absent, never an empty claim |
+| `.mutagent/evaluator/reports/<runId>/{evaluation,discovery,review}-report.html` | the renderers | the operator | the run wrote no report — fail loud |
+
+## Judge emissions (the verdict-file fields — E-NEW)
+
+| field | shape | honesty rule |
+|---|---|---|
+| `layerVerdicts[]` | `{layer, verdict, refs?, note?, scenario?, expectedExit?, gapKind?, firstProblemOrigin?, divergence?}` | skipped ≠ undecidable; L0 may be `fired` (a candidate, never a verdict) |
+| `verdicts[]` | per-criterion `{critique, result, confidence, refs?, assumptions?, blockedBy?}` | a DECIDED verdict carries ≥1 re-resolvable ref; an `uncertain` carries `blockedBy` (na for grounding) |
+| `denseMap` | `{criterionId: pass\|fail\|uncertain\|na}` | EVERY criterion appears; `na` ≠ fail |
+| `naRationale` | `{criterionId: why}` | the reason a cell is `na` — the completeness law's third path carries its rationale |
+| `emissions` | `{emitted[], missing[{key,reason}]}` | a missing emission WITH a stated reason is a NAMED degrade; a silent drop is a defect (MR-5 catches it) |
+| `codeEvalHits[]` | `{pattern, anchor, detail}` | L0 fired patterns — evidence, not a verdict |
+| `localize` | `{root, evidence?, criteria?, independentRoots?, conflict?, routing?, …}` | the root-not-symptom; `conflict` is the judge's OT-1 statement — surfaced, never reconciled |
+
+## Subject profile + criteria
+
+| entity | produced by | note |
+|---|---|---|
+| subject profile | `buildSubjectProfile` | identity · purpose · tools · scope — GIVEN (code access) or reconstructed |
+| `criteria[]` (the eval matrix) | the suite / `*discover` | each may carry `layer` (which evidence layer it binds to) — additive, optional, defaults to `"criteria"` |
+| living suite + provenance | `growLivingSuite` | append-only, monotonic; near-duplicates MERGE (no data loss) |
+| pending near-duplicate decisions | `merge-criteria` | durable, order-independent identity; stays open until a ruling |
+
+**The rule that binds every row of this registry:** an absence is stated, never inferred
+into a number. `0` and "not recorded" are different facts, and a report that shows the
+first when it means the second is manufacturing evidence.

package/.claude/skills/mutagent-evaluator/references/eval-layers.md

@@ -0,0 +1,52 @@
+# The evidence layers (L0–L4) — how the layered judge reads a trajectory
+
+> The know-how reference for the layered walk. Load when you need to reason about
+> **what a layer can and cannot decide**, why the judge descends, and what the
+> stopping rules permit. The judge emits one `LayerVerdict` per engaged layer; this
+> file is the map between a layer id and its meaning. Live consumers: the judge's
+> `layerVerdicts`, the aggregate's `foldLayerVerdicts` / `detectLayerConflicts`, and
+> the report's layer matrix.
+
+## The ladder
+
+The judge walks **cheapest-first, descending only when evidence requires**. Each layer
+answers a narrower question than the one above; a layer's verdict is *evidence*, never
+the run verdict on its own (the GATE is driven by criteria severity — D2).
+
+| layer | name | question it can settle | what it CANNOT decide (the fence) |
+|---|---|---|---|
+| **L0** | code checks | did a deterministic pattern fire? (fault passthrough · error output · malformed structure · unguarded send · out-of-order gate) | a fired pattern is a **candidate**, not a verdict — "code-detectable ≠ incorrect". Emits `codeEvalHits`, verdict `fired` — never pass/fail. |
+| **L1** | outcome | did the agent's terminal action belong to the scenario's expected exit set? (scenario → `expectedExit[]`) | anything about the *path*: a correct outcome can hide an unguarded send, a skipped review, an ignored flag. |
+| **L2** | trajectory | did the actual decision path match the expected one? (actual-vs-expected, read from the END) | whether a *tool's* output was trustworthy — that is L3. A divergence here is the first sign the outcome misleads. |
+| **L3** | tool outputs | were the tool outputs structurally valid + complete? (per-observation) | **external-data correctness is UNVERIFIABLE** — a well-formed output can be factually wrong. A failed tool makes everything downstream suspect. |
+| **L4** | context | was required context retrieved AND present in history at the moment of the decision? (chronology-sorted) | distinguish **`gapKind: retrieval`** (never fetched) from **`attention`** (present, ignored) — the fixes differ. |
+
+## Criteria are the super-layer
+
+Criteria do not live *in* one layer — they are the **super-layer** and may bind evidence
+from any phase (e.g. a reply-grounding criterion reads L3 tool outputs AND L1 the reply).
+Two hard rules:
+
+- **a criterion may NEVER go unverdicted because the agent did the right thing** — `na`
+  (not-applicable) is a real accounting path with a rationale, never a silent skip
+  (`assertNoUnverdictedCriterion` THROWS on a missing cell).
+- **`na` is not `uncertain`.** `na` = the criterion's scoping predicate is positively
+  falsified for this trajectory (never fail). `uncertain` = the inputs could not decide
+  (a CRIT/HIGH uncertain rolls the run INCOMPLETE-wards, never green).
+
+## The stopping rules (when the walk may exit early)
+
+- If **L1 passes AND every criterion is answerable** from evidence gathered so far → the
+  walk may set `earlyExit: 'L1'` and mark L2–L4 `skipped`. `skipped ≠ undecidable`.
+- If **any** criterion is unanswerable, OR L1 fails / is undecidable → **descend.** A
+  walk must not stop while a criterion is unanswerable (that is how a false-green forms).
+- `layersEngaged` records how deep the walk actually went (the economy stat — MR-3).
+
+## Conflict discipline (OT-1)
+
+When layers disagree — e.g. **L1 pass but L2 fail** (the outcome was right, the path
+unguarded) — the judge emits **both** verdicts honestly with a note and **never
+reconciles them itself**. The aggregate (`detectLayerConflicts`) surfaces the conflict
+to a human calibration queue; the GATE never reads it (it is driven by criteria
+severity). The judge's own statement of the tension lives in `localize.conflict`; the
+aggregate's cross-check is the layer notes. Both render; neither is auto-resolved.

package/.claude/skills/mutagent-evaluator/references/eval-stage.md

@@ -21,7 +21,7 @@ spine (`error-analysis.md` · `write-judge-prompt.md` · `validate-evaluator.md`
 | Input | Shape | Used for |
 |-------|-------|----------|
 | the built agent | a target-framework impl (e.g. a Mastra/TS agent) OR a harness agent | the eval SUBJECT |
-| `agentspec.definition.evals` | `{ success_criteria[], scenarios[], dataset_categories[] }` | dataset materialization (F8) + criteria |
+| `agentspec.spec.evaluation` | `{ criteria[], scenarios[], datasets[] }` (0.3.0, D18/D19 — datasets carry dataset-local `categories[]` + `caseDimensions` + real `items[]`) | dataset materialization (F8) + criteria |
 | `agentspec.build` | `{ target_framework, runtime, target_eval_framework }` | the target-conditional engine fork (F9) |
 
 The skill is **standalone** — it NEVER imports the agentspec schema. It consumes

package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts

@@ -17,15 +17,22 @@
  *         categories[] : { name, definition, class, fixOrEval, exampleTraceIds, … }
  *
  * AGGREGATE JOINS them into `TraceAnnotation[]` (NO hand-rolled annotations in
- * the path), then `deriveMinedCriteria` (§5b metadata + §5c DR-2) → `growLivingSuite`
+ * the path), MERGES near-duplicate categories (`merge-criteria.ts` — deterministic,
+ * no LLM), then `deriveMinedCriteria` (§5b metadata + §5c DR-2) → `growLivingSuite`
  * (append-only, monotonic). FAIL-LOUD: a missing verdict file (mirrors
  * `missingVerdictKeys`) or a category referencing an undispatched trace THROWS —
  * a dispatch gap is never a silently-fabricated annotation.
  *
+ * The merge runs BEFORE derivation on purpose: relabelling an absorbed category's
+ * annotations onto its survivor lets support / prevalence / refs / grounding be
+ * recomputed from the real union of per-trace evidence, rather than adding up
+ * already-derived strings. Nothing is dropped — a near-duplicate pair the guards
+ * refuse to merge is REPORTED (`mergePlan.findings`), not discarded.
+ *
  * PURE except for fs reads (the verdict files). No clock / random / network — the
  * mined criteria + suite are byte-identical for the same inputs (C-PIN).
  */
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { type Static, Type } from "@sinclair/typebox";
 import { Value } from "@sinclair/typebox/value";
@@ -44,8 +51,12 @@ import type { DatasetCase } from "./contracts/dataset.ts";
 import type { CandidateItem, MatrixVerdictFile } from "./contracts/eval-matrix.ts";
 import {
   CodeEvalSpecSchema,
+  Generality,
   JudgeKind,
+  MetricDimension,
+  MetricLevel,
   OutcomeVerdict,
+  Severity,
   assertGroundingHonest,
   type DiscoveryRef,
   type EvalTrace,
@@ -55,6 +66,16 @@ import {
 } from "./contracts/eval-types.ts";
 import type { CodeEvalSpec } from "./code-eval.ts";
 import { applyDiffDiscrimination, type CriterionFireSignals } from "./diff-discriminate.ts";
+import {
+  applyCriterionMerges,
+  buildExistingSuiteBrief,
+  emptyNearDuplicateLedger,
+  planCriterionMerges,
+  recordNearDuplicateFindings,
+  stampMergeProvenance,
+  type MergePlan,
+  type NearDuplicateLedger,
+} from "./merge-criteria.ts";
 
 // ── The leaf's per-batch mining report (real #mode-discover output shape) ────
 
@@ -66,6 +87,29 @@ export const DiscoverLabelSchema = Type.Object(
       Type.Literal(OutcomeVerdict.Fail),
       Type.Literal(OutcomeVerdict.Uncertain),
     ]),
+    /**
+     * The ROOT cause of the failure — NOT the first visible symptom.
+     *
+     * The name is historical and the SEMANTICS OVERRIDE IT: per the
+     * `root_not_symptom` invariant the leaf traces each failure to its root with
+     * judgement, because the first thing that looks wrong is usually downstream
+     * of the real cause. Read this field as "root cause"; keep one criterion per
+     * ROOT, and let independent roots produce separate criteria.
+     *
+     * ⚠️ DO NOT RENAME to `rootCause` casually. That rename was decided once,
+     * never executed, and left the agent contract asserting a field this closed
+     * schema rejects — a leaf that followed the contract literally hard-threw the
+     * whole AGGREGATE (`Expected required property firstThingWrong`, plus
+     * `additionalProperties:false` refusing the extra key). The contract was
+     * deliberately aligned to this shipped name RATHER THAN performing the rename,
+     * because the rename is a BREAKING change to the on-disk leaf-artifact
+     * contract — every stored mining report would need migrating — while the
+     * intent it was meant to express already lives in the `root_not_symptom`
+     * invariant, in the workflow step "Trace each Fail to its ROOT", and in this
+     * comment. The field NAME was never what carried the semantics. If the rename
+     * is genuinely wanted, schedule it as its own breaking change WITH an artifact
+     * migration; do not slip it into unrelated work.
+     */
     firstThingWrong: Type.String(),
     evidencePointer: Type.String(),
     /** GA-1 — optional STRUCTURED grounding refs `{obs,path,value}` the leaf
@@ -92,10 +136,40 @@ export const DiscoverCategorySchema = Type.Object(
     fixOrEval: Type.Union([Type.Literal("eval-worthy"), Type.Literal("fixable")]),
     exampleTraceIds: Type.Array(Type.String({ minLength: 1 })),
     // OPTIONAL §5b metadata proposals (additive agent-contract extension).
-    dimension: Type.Optional(Type.String({ minLength: 1 })),
-    level: Type.Optional(Type.String({ minLength: 1 })),
-    generality: Type.Optional(Type.String({ minLength: 1 })),
-    severity: Type.Optional(Type.String({ minLength: 1 })),
+    //
+    // CLOSED UNIONS, not free strings. These are the leaf's PROPOSALS and
+    // `parseDiscoverAnnotations` carries them through to the mined criterion, so
+    // a typo (`level: "outputs"`) must fail LOUD at parse time rather than be
+    // silently ignored — silently ignoring is precisely the bug that pinned every
+    // mined criterion to `level: output` and made layer-coverage unreadable.
+    dimension: Type.Optional(
+      Type.Union([
+        Type.Literal(MetricDimension.OperationCorrectness),
+        Type.Literal(MetricDimension.DataCorrectness),
+        Type.Literal(MetricDimension.OperationalDeviation),
+      ]),
+    ),
+    level: Type.Optional(
+      Type.Union([
+        Type.Literal(MetricLevel.Context),
+        Type.Literal(MetricLevel.Output),
+        Type.Literal(MetricLevel.CrossStage),
+      ]),
+    ),
+    generality: Type.Optional(
+      Type.Union([
+        Type.Literal(Generality.GeneralStructural),
+        Type.Literal(Generality.SpecificSemantic),
+      ]),
+    ),
+    severity: Type.Optional(
+      Type.Union([
+        Type.Literal(Severity.Crit),
+        Type.Literal(Severity.High),
+        Type.Literal(Severity.Med),
+        Type.Literal(Severity.Low),
+      ]),
+    ),
     judgeInputs: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
     /** UNIFORM STANDARD — the executable code-check the leaf emits for a
      *  deterministically-checkable category (validated against the canonical
@@ -218,6 +292,17 @@ export function parseDiscoverAnnotations(input: AggregateDiscoverInput): TraceAn
       if (meta !== undefined && meta.evidencePointer.length > 0) ann.evidencePointer = meta.evidencePointer;
       if (meta !== undefined && meta.refs !== undefined && meta.refs.length > 0) ann.refs = meta.refs;
       if (cat.judgeInputs !== undefined) ann.judgeInputs = cat.judgeInputs;
+      // §5b PROPOSALS — carry the leaf's OWN metadata through to the annotation.
+      // `deriveMinedCriteria` treats these as the value and its heuristics as the
+      // FALLBACK (`firstDefined(list, key) ?? derive(...)`), so present ⇒ honored,
+      // absent ⇒ derived. Before this, the schema accepted them and the JOIN
+      // dropped them on the floor: every mined criterion came out `level: output`
+      // with a fail-count-derived severity no matter what the leaf proposed, and
+      // the report's layer-coverage could never read more than one layer.
+      if (cat.level !== undefined) ann.level = cat.level;
+      if (cat.severity !== undefined) ann.severity = cat.severity;
+      if (cat.dimension !== undefined) ann.dimension = cat.dimension;
+      if (cat.generality !== undefined) ann.generality = cat.generality;
       // UNIFORM STANDARD — carry the leaf's executable code-check through to the
       // annotation so deriveMinedCriteria lands it on MinedCriterion.codeEval.
       if (cat.codeEval !== undefined) ann.codeEval = cat.codeEval as CodeEvalSpec;
@@ -229,12 +314,38 @@ export function parseDiscoverAnnotations(input: AggregateDiscoverInput): TraceAn
 
 // ── Full AGGREGATE: annotations → mined criteria → living-suite ──────────────
 
+/**
+ * How a mined criterion relates to the suite that existed BEFORE this run.
+ * `merged` is NOT a member: a criterion can be new-and-merged or
+ * reinforced-and-merged, so the merge is carried separately on
+ * `MinedCriterion.mergedFrom` and the report shows both facts.
+ */
+export const DiscoverCriterionState = {
+  /** minted by this run — no criterion with this id existed before. */
+  New: "new",
+  /** already in the suite; this run attached FURTHER evidence to it. */
+  Reinforced: "reinforced",
+} as const;
+export type DiscoverCriterionStateValue =
+  (typeof DiscoverCriterionState)[keyof typeof DiscoverCriterionState];
+
 export interface AggregateDiscoverResult {
   annotations: TraceAnnotation[];
   criteria: MinedCriterion[];
   suite: LivingSuite<MinedCriterion>;
   /** T6 — failure/uncertain DATASET CANDIDATES (present iff `traces` supplied). */
   datasetCandidates: DatasetCase[];
+  /** the near-duplicate MERGE plan actually applied (merges + surfaced findings). */
+  mergePlan: MergePlan;
+  /** criterion id → new | reinforced, judged against the PRE-run suite. */
+  criterionStates: Record<string, DiscoverCriterionStateValue>;
+  /**
+   * The DURABLE near-duplicate decision ledger, grown by this run. Pairs the
+   * guards refused to merge live here across runs with a stable identity until a
+   * human rules — "nothing destroyed, nothing forgotten". Persist it with
+   * `writeNearDuplicateLedger` and feed it back in on the next run.
+   */
+  nearDuplicateLedger: NearDuplicateLedger;
 }
 
 /**
@@ -258,9 +369,27 @@ export function aggregateDiscover(
     suite: LivingSuite<MinedCriterion>;
     traces?: EvalTrace[];
     fireSignals?: CriterionFireSignals[];
+    /** near-duplicate merge floor; omit for the measured default (0.4). */
+    mergeThreshold?: number;
+    /**
+     * The PRIOR durable near-duplicate ledger (from the last run). Omit on a
+     * first run. Supplying it is what stops an already-surfaced pair from being
+     * re-proposed as a fresh discovery every run.
+     */
+    nearDuplicateLedger?: NearDuplicateLedger;
   },
 ): AggregateDiscoverResult {
-  const annotations = parseDiscoverAnnotations(input);
+  const parsed = parseDiscoverAnnotations(input);
+
+  // Near-duplicate MERGE — BEFORE derivation, so support / prevalence / refs /
+  // grounding are all recomputed over the real union of per-trace evidence
+  // instead of hand-recombining derived strings. Blocked pairs are surfaced on
+  // `mergePlan.findings`, never dropped and never silently merged. With no
+  // merges, `applyCriterionMerges` returns the input untouched (zero change).
+  const mergePlan = planCriterionMerges(parsed, {
+    ...(input.mergeThreshold !== undefined ? { threshold: input.mergeThreshold } : {}),
+  });
+  const annotations = applyCriterionMerges(parsed, mergePlan);
   let criteria = deriveMinedCriteria(annotations);
 
   // GA-11 — observed-eligibility via broken∧¬healthy diff (deterministic).
@@ -273,6 +402,20 @@ export function aggregateDiscover(
     for (const c of criteria) assertGroundingHonest(c, input.traces);
   }
 
+  // MERGE provenance — `mergedFrom` (what was absorbed) + `aliases` (the absorbed
+  // ids kept resolvable, so an older report / dataset recipe / handoff naming a
+  // pre-merge id still finds its criterion).
+  criteria = stampMergeProvenance(criteria, mergePlan);
+
+  // NEW vs REINFORCED, judged against the suite as it stood BEFORE this run.
+  const knownBefore = new Set(input.suite.entries.map((c) => c.id));
+  const criterionStates: Record<string, DiscoverCriterionStateValue> = {};
+  for (const c of criteria) {
+    criterionStates[c.id] = knownBefore.has(c.id)
+      ? DiscoverCriterionState.Reinforced
+      : DiscoverCriterionState.New;
+  }
+
   const suite = growLivingSuite(input.suite, criteria, (c) => c.id);
 
   // T6 — surface failure/uncertain DATASET CANDIDATES (reusing the EV-047 selectors)
@@ -282,7 +425,23 @@ export function aggregateDiscover(
       ? collectDatasetCandidates({ determinerSpecs: input.determinerSpecs, verdictDir: input.verdictDir, traces: input.traces })
       : [];
 
-  return { annotations, criteria, suite, datasetCandidates };
+  // The undecidable band becomes DURABLE state, not a drawing: a pair a guard
+  // refused to merge is recorded (stable id, monotonic, rulings preserved) so it
+  // survives the reader closing the report and is not re-derived as new next run.
+  const nearDuplicateLedger = recordNearDuplicateFindings(
+    input.nearDuplicateLedger ?? emptyNearDuplicateLedger(),
+    mergePlan.findings,
+  );
+
+  return {
+    annotations,
+    criteria,
+    suite,
+    datasetCandidates,
+    mergePlan,
+    criterionStates,
+    nearDuplicateLedger,
+  };
 }
 
 // ── Report wiring: AGGREGATE result → report.html (mirrors writeRunReport) ───
@@ -343,6 +502,84 @@ export function writeDiscoverReport(input: WriteDiscoverReportInput): string {
   return outPath;
 }
 
+// ── PREP-side: hand the EXISTING suite to the mining leaf (prevention) ───────
+
+/** The file the `#mode-discover` leaf pre-reads to see what already exists. */
+export const EXISTING_CRITERIA_FILE = "existing-criteria.json";
+
+/**
+ * Write the existing living suite into the run dir as `existing-criteria.json`
+ * so the dispatched `#mode-discover` leaf can see WHAT ALREADY EXISTS before it
+ * clusters. Prevention, where the merge above is only cleanup: a leaf that
+ * recognises a behaviour it already has a criterion for attaches the trace to
+ * that criterion's id as REINFORCING evidence instead of minting a restatement
+ * under a new name.
+ *
+ * INFORMS, never blocks (the agent contract is explicit): observing the same
+ * behaviour again is legitimate evidence, and the leaf still mints a genuinely
+ * new criterion whenever the traces show one. An EMPTY suite still writes the
+ * file — "nothing exists yet" is information, and a missing file would be
+ * indistinguishable from a wiring gap. Returns the written path.
+ */
+export function writeExistingSuiteBrief(
+  dir: string,
+  suite: LivingSuite<MinedCriterion>,
+  ledger?: NearDuplicateLedger,
+): string {
+  mkdirSync(dir, { recursive: true });
+  const outPath = join(dir, EXISTING_CRITERIA_FILE);
+  writeFileSync(outPath, `${JSON.stringify(buildExistingSuiteBrief(suite, ledger), null, 2)}\n`);
+  return outPath;
+}
+
+// ── The durable near-duplicate decision ledger (cross-run, like the suite) ───
+
+/** The ledger filename. CROSS-RUN — lives beside the living suite, not under a run id. */
+export const NEAR_DUPLICATE_LEDGER_FILE = "near-duplicate-decisions.json";
+
+/**
+ * Read the durable near-duplicate ledger from `dir`, or an EMPTY ledger when it
+ * does not exist yet (a first run legitimately has none). A malformed ledger
+ * THROWS rather than being silently reset — quietly starting over would drop
+ * every open question and every ruling, which is precisely the
+ * "nothing forgotten" guarantee this file exists to keep.
+ */
+export function readNearDuplicateLedger(dir: string): NearDuplicateLedger {
+  const path = join(dir, NEAR_DUPLICATE_LEDGER_FILE);
+  if (!existsSync(path)) return emptyNearDuplicateLedger();
+  const raw = readFileSync(path, "utf8");
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(
+      `readNearDuplicateLedger: ${path} is not valid JSON (${err instanceof Error ? err.message : String(err)}). ` +
+        "Refusing to silently start a fresh ledger — that would forget every open near-duplicate " +
+        "question AND every human ruling. Fix or explicitly remove the file.",
+    );
+  }
+  const l = parsed as Partial<NearDuplicateLedger>;
+  if (!Array.isArray(l.decisions) || typeof l.version !== "number") {
+    throw new Error(
+      `readNearDuplicateLedger: ${path} is not a NearDuplicateLedger ({decisions[], version}). ` +
+        "Refusing to silently start over (see above).",
+    );
+  }
+  return l as NearDuplicateLedger;
+}
+
+/**
+ * Persist the durable near-duplicate ledger. CROSS-RUN state (mirrors the living
+ * suite's location, NOT the run dir) — the whole point is that it outlives the
+ * run that surfaced it. Returns the written path.
+ */
+export function writeNearDuplicateLedger(dir: string, ledger: NearDuplicateLedger): string {
+  mkdirSync(dir, { recursive: true });
+  const outPath = join(dir, NEAR_DUPLICATE_LEDGER_FILE);
+  writeFileSync(outPath, `${JSON.stringify(ledger, null, 2)}\n`);
+  return outPath;
+}
+
 // ── T6 (B-U5) discover → DATASET CANDIDATES + unmatched-detection handoff ────
 //
 // `*discover-evals` mined CRITERIA only; T6 ALSO surfaces a failure/uncertain DATASET
@@ -449,6 +686,10 @@ export interface AggregateDiscoverCliResult {
   suiteTotal: number;
   lastAppended: number;
   criterionIds: string[];
+  /** near-duplicate merges applied: `survivor <- absorbed, …` (empty when none). */
+  merged: string[];
+  /** near-duplicate pairs deliberately NOT merged, surfaced for review. */
+  nearDuplicateFindings: string[];
   /** present iff a `report.html` out path was supplied (5th arg). */
   reportPath?: string;
   /** whether the GA-1 ground-gate ran in this CLI invocation (traces supplied). */
@@ -483,7 +724,7 @@ export function runAggregateDiscoverCli(
       ? (JSON.parse(io.readFile(tracesPath)) as EvalTrace[])
       : undefined;
 
-  const { criteria, suite } = aggregateDiscover({
+  const { criteria, suite, mergePlan } = aggregateDiscover({
     determinerSpecs,
     verdictDir,
     miningReport,
@@ -512,6 +753,10 @@ export function runAggregateDiscoverCli(
     suiteTotal: suite.provenance.total,
     lastAppended: suite.provenance.lastAppended,
     criterionIds: criteria.map((c) => c.id),
+    merged: mergePlan.merges.map((m) => `${m.survivor} <- ${m.absorbed.join(", ")}`),
+    nearDuplicateFindings: mergePlan.findings.map(
+      (f) => `${f.a} ~ ${f.b} (${f.similarity.toFixed(2)}) NOT merged: ${f.kind}`,
+    ),
     ...(writtenReport !== undefined ? { reportPath: writtenReport } : {}),
     groundGateRan: traces !== undefined && traces.length > 0,
   };

package/.claude/skills/mutagent-evaluator/scripts/audience.ts

@@ -0,0 +1,84 @@
+/**
+ * audience — the V11 report AUDIENCE CONTRACT, shared by every v3 report renderer.
+ *
+ * WHY THIS EXISTS. `render-eval-report-v3` declared `audience: "internal" | "external"`
+ * and NEVER READ IT, so an external render was byte-identical to an internal one; the
+ * discovery renderer had no audience concept at all. Both therefore shipped their
+ * INTERNAL methodology surface to clients. This module holds the ONE implementation so
+ * the rule cannot drift between surfaces — and so the review surface, when it lands,
+ * inherits it instead of rediscovering it.
+ *
+ * ── WHAT AN `external` RENDER STRIPS ────────────────────────────────────────────
+ *   1. the INTERNAL methodology surface — the eval report's ⑤ Self-Eval (MR-1..5
+ *      self-grading) and the discovery report's ⑥ Methodology (process-as-executed,
+ *      provenance, proof-of-work, honest gaps) — INCLUDING:
+ *        · its TAB-BAR entry (never leave a tab that opens onto nothing), and
+ *        · its HTML COMMENT marker. A comment ships to the client exactly like any
+ *          other byte, so removing the visible tab and leaving `<!-- ⑤ SELF-EVAL -->`
+ *          behind is still a leak. This is not hypothetical: it is the case the
+ *          verifier's comment-blind matching caught.
+ *   2. run-internal OPERATIONAL detail — the pinned judge-model string, internal run
+ *      ids, and the gitignored artifact path printed in the footer.
+ *
+ * ── WHAT IT KEEPS ───────────────────────────────────────────────────────────────
+ *   verdicts · quoted evidence · findings · criteria · the gate. The client gets the
+ *   judgment and the proof, not how the judge graded itself. The determinism CLAIM
+ *   survives (rendered as "pinned · temp 0") because reproducibility is
+ *   client-relevant; the model identifier is not.
+ *
+ * ── ORDERING (a real trap) ──────────────────────────────────────────────────────
+ *   Strip AFTER slot-fill. The internal panel contains a data slot, so cutting it
+ *   first trips the renderer's fail-loud "TEMPLATE SLOT MISSING" guard. Fill every
+ *   slot, then remove the surface.
+ *
+ * Enforced browser-free by `verify-render.ts`, in BOTH directions — an external
+ * render must contain none of the internal markers, an internal render must still
+ * contain them. A one-directional check passes vacuously the moment a strip
+ * over-reaches or the renderer emits nothing.
+ *
+ * PURE — no clock, no randomness, no network, no filesystem.
+ */
+
+export type Audience = "internal" | "external";
+
+/** True when this render is for a CLIENT audience and must carry no internal surface. */
+export function isExternal(input: { audience?: Audience }): boolean {
+  return input.audience === "external";
+}
+
+/**
+ * Cut `startAnchor` … `endAnchor` out of `html`. FAIL-LOUD when either anchor is
+ * missing — a silent no-op here would ship the very surface being removed, which is
+ * the worst possible failure mode for a leak strip.
+ *
+ * `consumeEnd: false` stops immediately BEFORE the end anchor — needed when the anchor
+ * is markup that must survive (e.g. the container's closing tag after the last panel).
+ *
+ * NOTE on choosing an end anchor: do not reach for the panel's own closing tags. The
+ * cards inside these panels contain the same `</div></div>` sequence, so a first-match
+ * end anchor cuts one card and leaves the rest. Anchor on something UNIQUE that follows
+ * the panel (the footer marker) and stop before it.
+ */
+export function cutBlock(html: string, startAnchor: string, endAnchor: string, what: string, consumeEnd = true): string {
+  const start = html.indexOf(startAnchor);
+  const end = start >= 0 ? html.indexOf(endAnchor, start) : -1;
+  if (start < 0 || end < 0) throw new Error(`AUDIENCE STRIP anchor missing: ${what}`);
+  return html.slice(0, start) + html.slice(consumeEnd ? end + endAnchor.length : end);
+}
+
+/** The unique marker that follows the LAST panel in both v3 templates. */
+export const FOOT_ANCHOR = '  </div>\n\n  <div class="foot">';
+
+/**
+ * Strip an internal methodology surface: its comment-marked panel plus its tab-bar
+ * entry. `tabIndex` is the `onclick="tab(N)"` index of the entry to remove — the
+ * internal surface is the LAST tab on both surfaces, so the remaining indices stay
+ * contiguous and the template's index-based `tab()` switcher keeps working.
+ */
+export function stripInternalSurface(
+  html: string,
+  opts: { commentMarker: string; tabIndex: number; label: string },
+): string {
+  const withoutPanel = cutBlock(html, opts.commentMarker, FOOT_ANCHOR, `${opts.label} panel`, false);
+  return cutBlock(withoutPanel, `<s onclick="tab(${opts.tabIndex})">`, "</s>", `${opts.label} tab-bar entry`);
+}