@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts

@@ -17,15 +17,22 @@
  *         categories[] : { name, definition, class, fixOrEval, exampleTraceIds, … }
  *
  * AGGREGATE JOINS them into `TraceAnnotation[]` (NO hand-rolled annotations in
- * the path), then `deriveMinedCriteria` (§5b metadata + §5c DR-2) → `growLivingSuite`
+ * the path), MERGES near-duplicate categories (`merge-criteria.ts` — deterministic,
+ * no LLM), then `deriveMinedCriteria` (§5b metadata + §5c DR-2) → `growLivingSuite`
  * (append-only, monotonic). FAIL-LOUD: a missing verdict file (mirrors
  * `missingVerdictKeys`) or a category referencing an undispatched trace THROWS —
  * a dispatch gap is never a silently-fabricated annotation.
  *
+ * The merge runs BEFORE derivation on purpose: relabelling an absorbed category's
+ * annotations onto its survivor lets support / prevalence / refs / grounding be
+ * recomputed from the real union of per-trace evidence, rather than adding up
+ * already-derived strings. Nothing is dropped — a near-duplicate pair the guards
+ * refuse to merge is REPORTED (`mergePlan.findings`), not discarded.
+ *
  * PURE except for fs reads (the verdict files). No clock / random / network — the
  * mined criteria + suite are byte-identical for the same inputs (C-PIN).
  */
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { type Static, Type } from "@sinclair/typebox";
 import { Value } from "@sinclair/typebox/value";
@@ -44,8 +51,12 @@ import type { DatasetCase } from "./contracts/dataset.ts";
 import type { CandidateItem, MatrixVerdictFile } from "./contracts/eval-matrix.ts";
 import {
   CodeEvalSpecSchema,
+  Generality,
   JudgeKind,
+  MetricDimension,
+  MetricLevel,
   OutcomeVerdict,
+  Severity,
   assertGroundingHonest,
   type DiscoveryRef,
   type EvalTrace,
@@ -55,6 +66,16 @@ import {
 } from "./contracts/eval-types.ts";
 import type { CodeEvalSpec } from "./code-eval.ts";
 import { applyDiffDiscrimination, type CriterionFireSignals } from "./diff-discriminate.ts";
+import {
+  applyCriterionMerges,
+  buildExistingSuiteBrief,
+  emptyNearDuplicateLedger,
+  planCriterionMerges,
+  recordNearDuplicateFindings,
+  stampMergeProvenance,
+  type MergePlan,
+  type NearDuplicateLedger,
+} from "./merge-criteria.ts";
 
 // ── The leaf's per-batch mining report (real #mode-discover output shape) ────
 
@@ -66,6 +87,29 @@ export const DiscoverLabelSchema = Type.Object(
       Type.Literal(OutcomeVerdict.Fail),
       Type.Literal(OutcomeVerdict.Uncertain),
     ]),
+    /**
+     * The ROOT cause of the failure — NOT the first visible symptom.
+     *
+     * The name is historical and the SEMANTICS OVERRIDE IT: per the
+     * `root_not_symptom` invariant the leaf traces each failure to its root with
+     * judgement, because the first thing that looks wrong is usually downstream
+     * of the real cause. Read this field as "root cause"; keep one criterion per
+     * ROOT, and let independent roots produce separate criteria.
+     *
+     * ⚠️ DO NOT RENAME to `rootCause` casually. That rename was decided once,
+     * never executed, and left the agent contract asserting a field this closed
+     * schema rejects — a leaf that followed the contract literally hard-threw the
+     * whole AGGREGATE (`Expected required property firstThingWrong`, plus
+     * `additionalProperties:false` refusing the extra key). The contract was
+     * deliberately aligned to this shipped name RATHER THAN performing the rename,
+     * because the rename is a BREAKING change to the on-disk leaf-artifact
+     * contract — every stored mining report would need migrating — while the
+     * intent it was meant to express already lives in the `root_not_symptom`
+     * invariant, in the workflow step "Trace each Fail to its ROOT", and in this
+     * comment. The field NAME was never what carried the semantics. If the rename
+     * is genuinely wanted, schedule it as its own breaking change WITH an artifact
+     * migration; do not slip it into unrelated work.
+     */
     firstThingWrong: Type.String(),
     evidencePointer: Type.String(),
     /** GA-1 — optional STRUCTURED grounding refs `{obs,path,value}` the leaf
@@ -92,10 +136,40 @@ export const DiscoverCategorySchema = Type.Object(
     fixOrEval: Type.Union([Type.Literal("eval-worthy"), Type.Literal("fixable")]),
     exampleTraceIds: Type.Array(Type.String({ minLength: 1 })),
     // OPTIONAL §5b metadata proposals (additive agent-contract extension).
-    dimension: Type.Optional(Type.String({ minLength: 1 })),
-    level: Type.Optional(Type.String({ minLength: 1 })),
-    generality: Type.Optional(Type.String({ minLength: 1 })),
-    severity: Type.Optional(Type.String({ minLength: 1 })),
+    //
+    // CLOSED UNIONS, not free strings. These are the leaf's PROPOSALS and
+    // `parseDiscoverAnnotations` carries them through to the mined criterion, so
+    // a typo (`level: "outputs"`) must fail LOUD at parse time rather than be
+    // silently ignored — silently ignoring is precisely the bug that pinned every
+    // mined criterion to `level: output` and made layer-coverage unreadable.
+    dimension: Type.Optional(
+      Type.Union([
+        Type.Literal(MetricDimension.OperationCorrectness),
+        Type.Literal(MetricDimension.DataCorrectness),
+        Type.Literal(MetricDimension.OperationalDeviation),
+      ]),
+    ),
+    level: Type.Optional(
+      Type.Union([
+        Type.Literal(MetricLevel.Context),
+        Type.Literal(MetricLevel.Output),
+        Type.Literal(MetricLevel.CrossStage),
+      ]),
+    ),
+    generality: Type.Optional(
+      Type.Union([
+        Type.Literal(Generality.GeneralStructural),
+        Type.Literal(Generality.SpecificSemantic),
+      ]),
+    ),
+    severity: Type.Optional(
+      Type.Union([
+        Type.Literal(Severity.Crit),
+        Type.Literal(Severity.High),
+        Type.Literal(Severity.Med),
+        Type.Literal(Severity.Low),
+      ]),
+    ),
     judgeInputs: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
     /** UNIFORM STANDARD — the executable code-check the leaf emits for a
      *  deterministically-checkable category (validated against the canonical
@@ -218,6 +292,17 @@ export function parseDiscoverAnnotations(input: AggregateDiscoverInput): TraceAn
       if (meta !== undefined && meta.evidencePointer.length > 0) ann.evidencePointer = meta.evidencePointer;
       if (meta !== undefined && meta.refs !== undefined && meta.refs.length > 0) ann.refs = meta.refs;
       if (cat.judgeInputs !== undefined) ann.judgeInputs = cat.judgeInputs;
+      // §5b PROPOSALS — carry the leaf's OWN metadata through to the annotation.
+      // `deriveMinedCriteria` treats these as the value and its heuristics as the
+      // FALLBACK (`firstDefined(list, key) ?? derive(...)`), so present ⇒ honored,
+      // absent ⇒ derived. Before this, the schema accepted them and the JOIN
+      // dropped them on the floor: every mined criterion came out `level: output`
+      // with a fail-count-derived severity no matter what the leaf proposed, and
+      // the report's layer-coverage could never read more than one layer.
+      if (cat.level !== undefined) ann.level = cat.level;
+      if (cat.severity !== undefined) ann.severity = cat.severity;
+      if (cat.dimension !== undefined) ann.dimension = cat.dimension;
+      if (cat.generality !== undefined) ann.generality = cat.generality;
       // UNIFORM STANDARD — carry the leaf's executable code-check through to the
       // annotation so deriveMinedCriteria lands it on MinedCriterion.codeEval.
       if (cat.codeEval !== undefined) ann.codeEval = cat.codeEval as CodeEvalSpec;
@@ -229,12 +314,38 @@ export function parseDiscoverAnnotations(input: AggregateDiscoverInput): TraceAn
 
 // ── Full AGGREGATE: annotations → mined criteria → living-suite ──────────────
 
+/**
+ * How a mined criterion relates to the suite that existed BEFORE this run.
+ * `merged` is NOT a member: a criterion can be new-and-merged or
+ * reinforced-and-merged, so the merge is carried separately on
+ * `MinedCriterion.mergedFrom` and the report shows both facts.
+ */
+export const DiscoverCriterionState = {
+  /** minted by this run — no criterion with this id existed before. */
+  New: "new",
+  /** already in the suite; this run attached FURTHER evidence to it. */
+  Reinforced: "reinforced",
+} as const;
+export type DiscoverCriterionStateValue =
+  (typeof DiscoverCriterionState)[keyof typeof DiscoverCriterionState];
+
 export interface AggregateDiscoverResult {
   annotations: TraceAnnotation[];
   criteria: MinedCriterion[];
   suite: LivingSuite<MinedCriterion>;
   /** T6 — failure/uncertain DATASET CANDIDATES (present iff `traces` supplied). */
   datasetCandidates: DatasetCase[];
+  /** the near-duplicate MERGE plan actually applied (merges + surfaced findings). */
+  mergePlan: MergePlan;
+  /** criterion id → new | reinforced, judged against the PRE-run suite. */
+  criterionStates: Record<string, DiscoverCriterionStateValue>;
+  /**
+   * The DURABLE near-duplicate decision ledger, grown by this run. Pairs the
+   * guards refused to merge live here across runs with a stable identity until a
+   * human rules — "nothing destroyed, nothing forgotten". Persist it with
+   * `writeNearDuplicateLedger` and feed it back in on the next run.
+   */
+  nearDuplicateLedger: NearDuplicateLedger;
 }
 
 /**
@@ -258,9 +369,27 @@ export function aggregateDiscover(
     suite: LivingSuite<MinedCriterion>;
     traces?: EvalTrace[];
     fireSignals?: CriterionFireSignals[];
+    /** near-duplicate merge floor; omit for the measured default (0.4). */
+    mergeThreshold?: number;
+    /**
+     * The PRIOR durable near-duplicate ledger (from the last run). Omit on a
+     * first run. Supplying it is what stops an already-surfaced pair from being
+     * re-proposed as a fresh discovery every run.
+     */
+    nearDuplicateLedger?: NearDuplicateLedger;
   },
 ): AggregateDiscoverResult {
-  const annotations = parseDiscoverAnnotations(input);
+  const parsed = parseDiscoverAnnotations(input);
+
+  // Near-duplicate MERGE — BEFORE derivation, so support / prevalence / refs /
+  // grounding are all recomputed over the real union of per-trace evidence
+  // instead of hand-recombining derived strings. Blocked pairs are surfaced on
+  // `mergePlan.findings`, never dropped and never silently merged. With no
+  // merges, `applyCriterionMerges` returns the input untouched (zero change).
+  const mergePlan = planCriterionMerges(parsed, {
+    ...(input.mergeThreshold !== undefined ? { threshold: input.mergeThreshold } : {}),
+  });
+  const annotations = applyCriterionMerges(parsed, mergePlan);
   let criteria = deriveMinedCriteria(annotations);
 
   // GA-11 — observed-eligibility via broken∧¬healthy diff (deterministic).
@@ -273,6 +402,20 @@ export function aggregateDiscover(
     for (const c of criteria) assertGroundingHonest(c, input.traces);
   }
 
+  // MERGE provenance — `mergedFrom` (what was absorbed) + `aliases` (the absorbed
+  // ids kept resolvable, so an older report / dataset recipe / handoff naming a
+  // pre-merge id still finds its criterion).
+  criteria = stampMergeProvenance(criteria, mergePlan);
+
+  // NEW vs REINFORCED, judged against the suite as it stood BEFORE this run.
+  const knownBefore = new Set(input.suite.entries.map((c) => c.id));
+  const criterionStates: Record<string, DiscoverCriterionStateValue> = {};
+  for (const c of criteria) {
+    criterionStates[c.id] = knownBefore.has(c.id)
+      ? DiscoverCriterionState.Reinforced
+      : DiscoverCriterionState.New;
+  }
+
   const suite = growLivingSuite(input.suite, criteria, (c) => c.id);
 
   // T6 — surface failure/uncertain DATASET CANDIDATES (reusing the EV-047 selectors)
@@ -282,7 +425,23 @@ export function aggregateDiscover(
       ? collectDatasetCandidates({ determinerSpecs: input.determinerSpecs, verdictDir: input.verdictDir, traces: input.traces })
       : [];
 
-  return { annotations, criteria, suite, datasetCandidates };
+  // The undecidable band becomes DURABLE state, not a drawing: a pair a guard
+  // refused to merge is recorded (stable id, monotonic, rulings preserved) so it
+  // survives the reader closing the report and is not re-derived as new next run.
+  const nearDuplicateLedger = recordNearDuplicateFindings(
+    input.nearDuplicateLedger ?? emptyNearDuplicateLedger(),
+    mergePlan.findings,
+  );
+
+  return {
+    annotations,
+    criteria,
+    suite,
+    datasetCandidates,
+    mergePlan,
+    criterionStates,
+    nearDuplicateLedger,
+  };
 }
 
 // ── Report wiring: AGGREGATE result → report.html (mirrors writeRunReport) ───
@@ -343,6 +502,84 @@ export function writeDiscoverReport(input: WriteDiscoverReportInput): string {
   return outPath;
 }
 
+// ── PREP-side: hand the EXISTING suite to the mining leaf (prevention) ───────
+
+/** The file the `#mode-discover` leaf pre-reads to see what already exists. */
+export const EXISTING_CRITERIA_FILE = "existing-criteria.json";
+
+/**
+ * Write the existing living suite into the run dir as `existing-criteria.json`
+ * so the dispatched `#mode-discover` leaf can see WHAT ALREADY EXISTS before it
+ * clusters. Prevention, where the merge above is only cleanup: a leaf that
+ * recognises a behaviour it already has a criterion for attaches the trace to
+ * that criterion's id as REINFORCING evidence instead of minting a restatement
+ * under a new name.
+ *
+ * INFORMS, never blocks (the agent contract is explicit): observing the same
+ * behaviour again is legitimate evidence, and the leaf still mints a genuinely
+ * new criterion whenever the traces show one. An EMPTY suite still writes the
+ * file — "nothing exists yet" is information, and a missing file would be
+ * indistinguishable from a wiring gap. Returns the written path.
+ */
+export function writeExistingSuiteBrief(
+  dir: string,
+  suite: LivingSuite<MinedCriterion>,
+  ledger?: NearDuplicateLedger,
+): string {
+  mkdirSync(dir, { recursive: true });
+  const outPath = join(dir, EXISTING_CRITERIA_FILE);
+  writeFileSync(outPath, `${JSON.stringify(buildExistingSuiteBrief(suite, ledger), null, 2)}\n`);
+  return outPath;
+}
+
+// ── The durable near-duplicate decision ledger (cross-run, like the suite) ───
+
+/** The ledger filename. CROSS-RUN — lives beside the living suite, not under a run id. */
+export const NEAR_DUPLICATE_LEDGER_FILE = "near-duplicate-decisions.json";
+
+/**
+ * Read the durable near-duplicate ledger from `dir`, or an EMPTY ledger when it
+ * does not exist yet (a first run legitimately has none). A malformed ledger
+ * THROWS rather than being silently reset — quietly starting over would drop
+ * every open question and every ruling, which is precisely the
+ * "nothing forgotten" guarantee this file exists to keep.
+ */
+export function readNearDuplicateLedger(dir: string): NearDuplicateLedger {
+  const path = join(dir, NEAR_DUPLICATE_LEDGER_FILE);
+  if (!existsSync(path)) return emptyNearDuplicateLedger();
+  const raw = readFileSync(path, "utf8");
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(
+      `readNearDuplicateLedger: ${path} is not valid JSON (${err instanceof Error ? err.message : String(err)}). ` +
+        "Refusing to silently start a fresh ledger — that would forget every open near-duplicate " +
+        "question AND every human ruling. Fix or explicitly remove the file.",
+    );
+  }
+  const l = parsed as Partial<NearDuplicateLedger>;
+  if (!Array.isArray(l.decisions) || typeof l.version !== "number") {
+    throw new Error(
+      `readNearDuplicateLedger: ${path} is not a NearDuplicateLedger ({decisions[], version}). ` +
+        "Refusing to silently start over (see above).",
+    );
+  }
+  return l as NearDuplicateLedger;
+}
+
+/**
+ * Persist the durable near-duplicate ledger. CROSS-RUN state (mirrors the living
+ * suite's location, NOT the run dir) — the whole point is that it outlives the
+ * run that surfaced it. Returns the written path.
+ */
+export function writeNearDuplicateLedger(dir: string, ledger: NearDuplicateLedger): string {
+  mkdirSync(dir, { recursive: true });
+  const outPath = join(dir, NEAR_DUPLICATE_LEDGER_FILE);
+  writeFileSync(outPath, `${JSON.stringify(ledger, null, 2)}\n`);
+  return outPath;
+}
+
 // ── T6 (B-U5) discover → DATASET CANDIDATES + unmatched-detection handoff ────
 //
 // `*discover-evals` mined CRITERIA only; T6 ALSO surfaces a failure/uncertain DATASET
@@ -449,6 +686,10 @@ export interface AggregateDiscoverCliResult {
   suiteTotal: number;
   lastAppended: number;
   criterionIds: string[];
+  /** near-duplicate merges applied: `survivor <- absorbed, …` (empty when none). */
+  merged: string[];
+  /** near-duplicate pairs deliberately NOT merged, surfaced for review. */
+  nearDuplicateFindings: string[];
   /** present iff a `report.html` out path was supplied (5th arg). */
   reportPath?: string;
   /** whether the GA-1 ground-gate ran in this CLI invocation (traces supplied). */
@@ -483,7 +724,7 @@ export function runAggregateDiscoverCli(
       ? (JSON.parse(io.readFile(tracesPath)) as EvalTrace[])
       : undefined;
 
-  const { criteria, suite } = aggregateDiscover({
+  const { criteria, suite, mergePlan } = aggregateDiscover({
     determinerSpecs,
     verdictDir,
     miningReport,
@@ -512,6 +753,10 @@ export function runAggregateDiscoverCli(
     suiteTotal: suite.provenance.total,
     lastAppended: suite.provenance.lastAppended,
     criterionIds: criteria.map((c) => c.id),
+    merged: mergePlan.merges.map((m) => `${m.survivor} <- ${m.absorbed.join(", ")}`),
+    nearDuplicateFindings: mergePlan.findings.map(
+      (f) => `${f.a} ~ ${f.b} (${f.similarity.toFixed(2)}) NOT merged: ${f.kind}`,
+    ),
     ...(writtenReport !== undefined ? { reportPath: writtenReport } : {}),
     groundGateRan: traces !== undefined && traces.length > 0,
   };

package/.claude/skills/mutagent-evaluator/scripts/audience.ts

@@ -0,0 +1,84 @@
+/**
+ * audience — the V11 report AUDIENCE CONTRACT, shared by every v3 report renderer.
+ *
+ * WHY THIS EXISTS. `render-eval-report-v3` declared `audience: "internal" | "external"`
+ * and NEVER READ IT, so an external render was byte-identical to an internal one; the
+ * discovery renderer had no audience concept at all. Both therefore shipped their
+ * INTERNAL methodology surface to clients. This module holds the ONE implementation so
+ * the rule cannot drift between surfaces — and so the review surface, when it lands,
+ * inherits it instead of rediscovering it.
+ *
+ * ── WHAT AN `external` RENDER STRIPS ────────────────────────────────────────────
+ *   1. the INTERNAL methodology surface — the eval report's ⑤ Self-Eval (MR-1..5
+ *      self-grading) and the discovery report's ⑥ Methodology (process-as-executed,
+ *      provenance, proof-of-work, honest gaps) — INCLUDING:
+ *        · its TAB-BAR entry (never leave a tab that opens onto nothing), and
+ *        · its HTML COMMENT marker. A comment ships to the client exactly like any
+ *          other byte, so removing the visible tab and leaving `<!-- ⑤ SELF-EVAL -->`
+ *          behind is still a leak. This is not hypothetical: it is the case the
+ *          verifier's comment-blind matching caught.
+ *   2. run-internal OPERATIONAL detail — the pinned judge-model string, internal run
+ *      ids, and the gitignored artifact path printed in the footer.
+ *
+ * ── WHAT IT KEEPS ───────────────────────────────────────────────────────────────
+ *   verdicts · quoted evidence · findings · criteria · the gate. The client gets the
+ *   judgment and the proof, not how the judge graded itself. The determinism CLAIM
+ *   survives (rendered as "pinned · temp 0") because reproducibility is
+ *   client-relevant; the model identifier is not.
+ *
+ * ── ORDERING (a real trap) ──────────────────────────────────────────────────────
+ *   Strip AFTER slot-fill. The internal panel contains a data slot, so cutting it
+ *   first trips the renderer's fail-loud "TEMPLATE SLOT MISSING" guard. Fill every
+ *   slot, then remove the surface.
+ *
+ * Enforced browser-free by `verify-render.ts`, in BOTH directions — an external
+ * render must contain none of the internal markers, an internal render must still
+ * contain them. A one-directional check passes vacuously the moment a strip
+ * over-reaches or the renderer emits nothing.
+ *
+ * PURE — no clock, no randomness, no network, no filesystem.
+ */
+
+export type Audience = "internal" | "external";
+
+/** True when this render is for a CLIENT audience and must carry no internal surface. */
+export function isExternal(input: { audience?: Audience }): boolean {
+  return input.audience === "external";
+}
+
+/**
+ * Cut `startAnchor` … `endAnchor` out of `html`. FAIL-LOUD when either anchor is
+ * missing — a silent no-op here would ship the very surface being removed, which is
+ * the worst possible failure mode for a leak strip.
+ *
+ * `consumeEnd: false` stops immediately BEFORE the end anchor — needed when the anchor
+ * is markup that must survive (e.g. the container's closing tag after the last panel).
+ *
+ * NOTE on choosing an end anchor: do not reach for the panel's own closing tags. The
+ * cards inside these panels contain the same `</div></div>` sequence, so a first-match
+ * end anchor cuts one card and leaves the rest. Anchor on something UNIQUE that follows
+ * the panel (the footer marker) and stop before it.
+ */
+export function cutBlock(html: string, startAnchor: string, endAnchor: string, what: string, consumeEnd = true): string {
+  const start = html.indexOf(startAnchor);
+  const end = start >= 0 ? html.indexOf(endAnchor, start) : -1;
+  if (start < 0 || end < 0) throw new Error(`AUDIENCE STRIP anchor missing: ${what}`);
+  return html.slice(0, start) + html.slice(consumeEnd ? end + endAnchor.length : end);
+}
+
+/** The unique marker that follows the LAST panel in both v3 templates. */
+export const FOOT_ANCHOR = '  </div>\n\n  <div class="foot">';
+
+/**
+ * Strip an internal methodology surface: its comment-marked panel plus its tab-bar
+ * entry. `tabIndex` is the `onclick="tab(N)"` index of the entry to remove — the
+ * internal surface is the LAST tab on both surfaces, so the remaining indices stay
+ * contiguous and the template's index-based `tab()` switcher keeps working.
+ */
+export function stripInternalSurface(
+  html: string,
+  opts: { commentMarker: string; tabIndex: number; label: string },
+): string {
+  const withoutPanel = cutBlock(html, opts.commentMarker, FOOT_ANCHOR, `${opts.label} panel`, false);
+  return cutBlock(withoutPanel, `<s onclick="tab(${opts.tabIndex})">`, "</s>", `${opts.label} tab-bar entry`);
+}