@mutagent/evaluator 0.1.0-alpha.4 → 0.1.0-alpha.6

package/.claude/skills/mutagent-evaluator/scripts/memory/ratify.ts

@@ -0,0 +1,165 @@
+/**
+ * scripts/memory/ratify.ts — EV-4: persist ratify / do-not decisions + hold the verdict.
+ * ---------------------------------------------------------------------------
+ * The dominant dogfood finding ("router-as-doer") rested on a NORMATIVE call —
+ * what counts as correct routing — that only the operator can settle. The review
+ * UI already CAPTURES the operator's verify/eliminate decisions on each surfaced
+ * assumption (`build-review-ui.ts` `setCalib` → `calibration.json`), but wrote them
+ * NOWHERE: a verdict resting on an unratified standard still read as final.
+ *
+ * This is the LEAN wiring (decision EV-4 option a — NO new criterion schema):
+ *   1. ROUTE each calibration decision into `appendMemory` — the same store the
+ *      next run's judges read at start (`memory/read.ts`), deduped by slug.
+ *   2. HOLD the verdict: a NORMATIVE criterion with no `verify` ratification in the
+ *      decision set is `needs-ratification`; while any is pending the run's verdict
+ *      is HELD (not final) — the value call must be settled first.
+ *
+ * PURE cores (deterministic; `now` injected). The only fs edge reuses `appendMemory`.
+ */
+import { appendMemory, Lifecycle, MemoryType, type AppendResult } from "./append.ts";
+import { AssumptionKind, type CriterionVerdict } from "../contracts/eval-types.ts";
+
+/**
+ * The criterion ids whose verdict rests on a NORMATIVE assumption (a value/standard
+ * call — `AssumptionKind.Normative`), whether surfaced in `assumptions[]` or the
+ * `blockedBy` payload. These are the criteria that REQUIRE operator ratification
+ * before their verdict can be final. PURE.
+ */
+export function normativeCriterionIds(verdicts: CriterionVerdict[]): string[] {
+  const ids = new Set<string>();
+  for (const v of verdicts) {
+    const inAssumptions = (v.assumptions ?? []).some((a) => a.kind === AssumptionKind.Normative);
+    const inBlock = v.blockedBy?.kind === AssumptionKind.Normative;
+    if (inAssumptions || inBlock) ids.add(v.criterionId);
+  }
+  return [...ids].sort();
+}
+
+/** The verify/eliminate action a review-UI calibration button emits (`data-action`). */
+export const RatifyAction = {
+  /** the operator RATIFIES the surfaced assumption/standard (accept it). */
+  Verify: "verify",
+  /** the operator rejects it (do-not — drop this standard). */
+  Eliminate: "eliminate",
+} as const;
+export type RatifyActionValue = (typeof RatifyAction)[keyof typeof RatifyAction];
+
+/**
+ * One calibration decision as the review UI exports it (`calibration.json`). Shape
+ * mirrors `build-review-ui.ts` `setCalib` EXACTLY so the operator's export routes
+ * with no transform.
+ */
+export interface CalibrationDecision {
+  traceId: string;
+  criterionId: string;
+  assumptionIndex: number;
+  action: RatifyActionValue;
+  decidedAt?: string;
+}
+
+/** A stable slug for a ratification memory (dedupe: re-deciding the same pair updates). */
+export function ratificationSlug(d: CalibrationDecision): string {
+  return `ratify-${d.criterionId}-a${d.assumptionIndex}`;
+}
+
+/**
+ * Project a calibration decision → an AutoMemory FeedbackInput. A `verify` is a
+ * standing "this standard IS ratified" fact the next run's judge honours; an
+ * `eliminate` records the operator dropped it. PURE.
+ */
+export function calibrationToMemory(
+  d: CalibrationDecision,
+  criterionStatement?: string,
+): { title: string; description: string; body: string } {
+  const ratified = d.action === RatifyAction.Verify;
+  const what = criterionStatement !== undefined && criterionStatement.length > 0 ? criterionStatement : d.criterionId;
+  const title = ratificationSlug(d);
+  const verb = ratified ? "RATIFIED" : "REJECTED (do-not)";
+  return {
+    title,
+    description: `Normative ratification — criterion ${d.criterionId} assumption #${d.assumptionIndex} ${verb}`,
+    body:
+      `Operator ${verb} the normative standard for criterion \`${d.criterionId}\` ` +
+      `(assumption #${d.assumptionIndex}).\n\n` +
+      `**What:** ${what}\n` +
+      `**Why:** a normative criterion (what SHOULD count as good) is an operator value call — ` +
+      `the judge cannot settle it. This decision is the settled standard.\n` +
+      `**How to apply:** ${ratified
+        ? "treat this standard as ratified — the verdict on this criterion may now be final."
+        : "do NOT gate on this standard — the operator dropped it."}\n` +
+      (d.decidedAt !== undefined ? `\n(decided ${d.decidedAt})` : ""),
+  };
+}
+
+/**
+ * ROUTE calibration decisions into AutoMemory (EV-4). One `appendMemory` per
+ * decision, deduped by slug (re-deciding UPDATES). Returns the append results.
+ * `now` (YYYY-MM-DD) is INJECTED. `statements` optionally maps criterionId → its
+ * statement for a richer memory body.
+ */
+export function persistCalibrations(
+  memoryDir: string,
+  decisions: CalibrationDecision[],
+  now: string,
+  statements: Record<string, string> = {},
+): AppendResult[] {
+  return decisions.map((d) => {
+    const m = calibrationToMemory(d, statements[d.criterionId]);
+    return appendMemory(
+      memoryDir,
+      {
+        title: m.title,
+        description: m.description,
+        lifecycle: Lifecycle.Evaluate,
+        type: MemoryType.Feedback, // a standing steer the next run's judge honours
+        feedback: { text: m.body },
+      },
+      now,
+    );
+  });
+}
+
+/** The ratification state of one normative criterion (EV-4). */
+export interface RatificationStatus {
+  criterionId: string;
+  /** true when a `verify` decision ratified it; false ⇒ needs-ratification. */
+  ratified: boolean;
+}
+
+/**
+ * Given the NORMATIVE criterion ids (those resting on a normative assumption) and
+ * the operator's calibration decisions, report which are RATIFIED (a `verify`
+ * exists) vs still `needs-ratification`. An `eliminate` does NOT ratify — the
+ * standard was dropped, not settled-as-correct. PURE.
+ */
+export function ratificationStatus(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): RatificationStatus[] {
+  const ratified = new Set(
+    decisions.filter((d) => d.action === RatifyAction.Verify).map((d) => d.criterionId),
+  );
+  return normativeCriterionIds.map((id) => ({ criterionId: id, ratified: ratified.has(id) }));
+}
+
+/**
+ * The criterion ids that HOLD the verdict (EV-4): normative criteria not yet
+ * ratified. While this list is non-empty the run's verdict is NOT final — the
+ * operator must settle the value call first. PURE.
+ */
+export function pendingRatifications(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): string[] {
+  return ratificationStatus(normativeCriterionIds, decisions)
+    .filter((s) => !s.ratified)
+    .map((s) => s.criterionId);
+}
+
+/** Is the run's verdict HELD pending an operator ratification? (EV-4) */
+export function isVerdictHeldForRatification(
+  normativeCriterionIds: string[],
+  decisions: CalibrationDecision[],
+): boolean {
+  return pendingRatifications(normativeCriterionIds, decisions).length > 0;
+}

package/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts

@@ -0,0 +1,232 @@
+/**
+ * scripts/persist-eval-criteria.ts — the EVAL-LEG on-disk WRITER (Wave-2 FU57 · KP-003).
+ * ---------------------------------------------------------------------------
+ * The IO half of the eval-leg reconcile. `sync-eval-criteria.ts` COMPUTES the grown
+ * criteria set as DATA (`reconcileEvalCriteria` — pure upsert-by-id, monotonic, never
+ * drops); it never touches disk. This module is the deterministic WRITER the evaluator
+ * SESSION calls to PERSIST that computed set back to the located eval-suite criteria
+ * artifact — the write that bumps the artifact's freshness and thereby returns the eval
+ * leg of `#sync-spec` to `in-sync` (the builder freshness probe reads the artifact's
+ * mtime / `updatedAt`; a fresh write makes the eval criteria as new as the amended impl).
+ *
+ * SEPARATION OF CONCERNS (mirrors `artifact-paths.ts` + `aggregate-discover.ts` doing IO
+ * while `living-suite.ts` stays pure): the reconcile COMPUTE is pure/C-PIN; this WRITE is
+ * the IO layer. The SERIALIZATION here is still deterministic (stable key order, no clock
+ * embedded unless the caller passes an explicit `updatedAt`), so a persisted artifact
+ * round-trips byte-faithful.
+ *
+ * JUDGE-ONLY PRESERVED (EV-051): this file writes CRITERIA (the evaluator maintaining its
+ * own eval-suite / code-quality criteria set) — it NEVER scores a subject, NEVER decides a
+ * pass/fail, and NEVER dispatches an agent (Model-B: code is a deterministic writer; the
+ * reconcile REASONING is the ai-architect's, session-dispatched).
+ *
+ * LOCATION + FORMAT (mirror the living-suite persistence; FLAGGED-for-operator, see
+ * `EVAL_CRITERIA_ARTIFACT_FILENAMES`): the artifact lands under the evaluator's namespaced
+ * `.mutagent/evaluator/living-suite/` root (`artifact-paths.ts` `livingSuiteDir`, auto-
+ * gitignored, no-spillover-guarded) as one JSON file per LEG — the same dir + extension the
+ * builder `check-sync-spec.ts` `locateEvalCriteria` auto-discovers.
+ */
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+import { assertUnderRoot, livingSuiteDir } from "./artifact-paths.ts";
+import {
+  EvalCriterionSchema,
+  EvalLegKind,
+  reconcileEvalCriteria,
+  type EvalCriteriaReconcileRequest,
+  type EvalCriteriaReconcileResult,
+  type EvalCriterion,
+  type EvalLegKindValue,
+} from "./sync-eval-criteria.ts";
+
+// ── the on-disk artifact shape (mirror LivingSuite `{ …, provenance }`) ───────
+
+/**
+ * The persisted eval-criteria artifact — one per LEG. Mirrors the living-suite shape
+ * (a keyed collection + provenance), specialized to the reconcile result: the maintained
+ * `criteria` set, the reconcile `provenance` counters, and an OPTIONAL `updatedAt`
+ * freshness marker. `updatedAt` is the ONE clock-bearing field and it is NEVER generated
+ * here — the caller (the session, Model-B) passes it when it wants a content-level
+ * freshness stamp the builder probe's `parseEvalUpdatedAt` can read; omitted, freshness
+ * rides on the file's mtime alone (write bumps it). `additionalProperties: false` keeps the
+ * header tight while each criterion stays forward-parsing (EvalCriterionSchema is open).
+ */
+export const EvalCriteriaArtifactSchema = Type.Object(
+  {
+    subjectId: Type.String({ minLength: 1 }),
+    leg: Type.Union([
+      Type.Literal(EvalLegKind.EvalSuite),
+      Type.Literal(EvalLegKind.CodeQuality),
+    ]),
+    criteria: Type.Array(EvalCriterionSchema),
+    provenance: Type.Object(
+      { added: Type.Number(), updated: Type.Number(), total: Type.Number() },
+      { additionalProperties: false },
+    ),
+    updatedAt: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+);
+export type EvalCriteriaArtifact = Static<typeof EvalCriteriaArtifactSchema>;
+
+/**
+ * FLAGGED-for-operator: the per-leg artifact FILENAME under `.mutagent/evaluator/living-suite/`.
+ * The sensible default that mirrors the living-suite location AND the builder probe's
+ * conventional fallback names (`eval-criteria` / `code-quality-criteria`), one file per leg
+ * so the two legs never collide. Both are `.json` under the dir `check-sync-spec.ts`
+ * `locateEvalCriteria` walks (`*.{yaml,yml,json}`), so a written artifact is auto-discovered
+ * as the eval-leg freshness anchor with no extra wiring. (A workspace evaluates ONE subject,
+ * so only its leg's file exists; multi-subject-per-workspace would want a `<subjectId>/`
+ * sub-dir — deferred, flagged.)
+ */
+export const EVAL_CRITERIA_ARTIFACT_FILENAMES: Readonly<Record<EvalLegKindValue, string>> = {
+  [EvalLegKind.EvalSuite]: "eval-suite.criteria.json",
+  [EvalLegKind.CodeQuality]: "code-quality.criteria.json",
+};
+
+/**
+ * Resolve the located eval-criteria artifact path for a leg:
+ * `<cwd>/.mutagent/evaluator/living-suite/<leg>.criteria.json`. Routes through the
+ * `artifact-paths.ts` no-spillover guard (`assertUnderRoot`), so the writer can NEVER
+ * escape the evaluator's namespaced root. `cwd` passed in (pure path construction).
+ */
+export function evalCriteriaArtifactPath(leg: EvalLegKindValue, cwd?: string): string {
+  return assertUnderRoot(join(livingSuiteDir(cwd), EVAL_CRITERIA_ARTIFACT_FILENAMES[leg]), cwd);
+}
+
+// ── serialize / build (pure, deterministic — no clock) ───────────────────────
+
+/** Build the on-disk artifact from a reconcile RESULT. Pure. `updatedAt` passed in. */
+export function evalCriteriaArtifactFromResult(
+  result: EvalCriteriaReconcileResult,
+  updatedAt?: string,
+): EvalCriteriaArtifact {
+  return {
+    subjectId: result.subjectId,
+    leg: result.leg,
+    criteria: result.criteria,
+    provenance: result.provenance,
+    ...(updatedAt !== undefined ? { updatedAt } : {}),
+  };
+}
+
+/**
+ * Serialize an artifact to its on-disk JSON — DETERMINISTIC: explicit key order, 2-space
+ * indent, trailing newline. Same artifact → byte-identical string (round-trip faithful);
+ * no clock unless the artifact already carries a caller-supplied `updatedAt`.
+ */
+export function serializeEvalCriteriaArtifact(artifact: EvalCriteriaArtifact): string {
+  const ordered: EvalCriteriaArtifact = {
+    subjectId: artifact.subjectId,
+    leg: artifact.leg,
+    criteria: artifact.criteria,
+    provenance: artifact.provenance,
+    ...(artifact.updatedAt !== undefined ? { updatedAt: artifact.updatedAt } : {}),
+  };
+  return `${JSON.stringify(ordered, null, 2)}\n`;
+}
+
+/** Guarded parse of a persisted artifact. THROWS with the first schema error. PURE. */
+export function parseEvalCriteriaArtifact(value: unknown): EvalCriteriaArtifact {
+  if (!Value.Check(EvalCriteriaArtifactSchema, value)) {
+    const first = [...Value.Errors(EvalCriteriaArtifactSchema, value)][0];
+    throw new Error(
+      `parseEvalCriteriaArtifact: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid eval-criteria artifact"}`,
+    );
+  }
+  return value;
+}
+
+// ── the WRITER + reader (the IO layer) ───────────────────────────────────────
+
+/** The result of a persist: WHERE it landed + WHAT was written (artifact + bytes). */
+export interface PersistEvalCriteriaResult {
+  /** the located artifact path the criteria were written to. */
+  path: string;
+  /** the artifact that was serialized. */
+  artifact: EvalCriteriaArtifact;
+  /** the exact bytes written (for a byte-faithful round-trip assertion). */
+  json: string;
+}
+
+/**
+ * PERSIST a reconcile result's `criteria` back to the located eval-suite criteria artifact
+ * — the WRITE that returns the eval leg to `in-sync`. Resolves the per-leg path under
+ * `.mutagent/evaluator/living-suite/` (no-spillover-guarded via `artifact-paths.ts`),
+ * ensures the dir, and writes the deterministic JSON. Handles BOTH legs by `result.leg`:
+ * the eval-suite leg (agent/skill/composite subject) and the code-quality leg (a code
+ * subject, incl. an operator-overridden code-quality set). The freshness bump is the file
+ * write itself (mtime advances → the builder probe reads the eval leg as fresh); pass
+ * `options.updatedAt` (ISO 8601) to ALSO stamp a content-level freshness marker the probe's
+ * `parseEvalUpdatedAt` reads. IO — NOT pure (that is its job); serialization is deterministic.
+ */
+export function persistEvalCriteria(
+  result: EvalCriteriaReconcileResult,
+  options: { cwd?: string; updatedAt?: string } = {},
+): PersistEvalCriteriaResult {
+  const artifact = evalCriteriaArtifactFromResult(result, options.updatedAt);
+  const path = evalCriteriaArtifactPath(result.leg, options.cwd);
+  const json = serializeEvalCriteriaArtifact(artifact);
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, json);
+  return { path, artifact, json };
+}
+
+/** Read + validate a persisted artifact (round-trip). THROWS on a schema violation. */
+export function readEvalCriteriaArtifact(path: string): EvalCriteriaArtifact {
+  return parseEvalCriteriaArtifact(JSON.parse(readFileSync(path, "utf8")) as unknown);
+}
+
+/**
+ * Load the CURRENTLY-persisted criteria for a leg (the `existing` set the next reconcile
+ * grows). Returns `[]` when no artifact exists yet (the cold / first-reconcile case) so a
+ * caller can feed it straight into an `EvalCriteriaReconcileRequest.existing`. Reads only.
+ */
+export function loadExistingEvalCriteria(leg: EvalLegKindValue, cwd?: string): EvalCriterion[] {
+  const path = evalCriteriaArtifactPath(leg, cwd);
+  if (!existsSync(path)) return [];
+  return readEvalCriteriaArtifact(path).criteria;
+}
+
+/**
+ * The eval-criteria artifact's effective freshness epoch (seconds) — the evaluator-side
+ * MIRROR of the builder `check-sync-spec.ts` eval-leg compare: `max(updatedAt, mtime)`
+ * (git omitted — kept in-package + git-independent). `null` when the artifact is absent.
+ * Feed this as `evalCriteriaEpoch` to `flagEvalLegDrift` to confirm a fresh write closed
+ * the loop (`in-sync`). Reads mtime (IO).
+ */
+export function evalCriteriaEffectiveEpoch(path: string): number | null {
+  if (!existsSync(path)) return null;
+  const epochs: number[] = [];
+  try {
+    const parsed = readEvalCriteriaArtifact(path);
+    if (parsed.updatedAt !== undefined) {
+      const ms = Date.parse(parsed.updatedAt);
+      if (Number.isFinite(ms)) epochs.push(Math.floor(ms / 1000));
+    }
+  } catch {
+    // a corrupt artifact still has an mtime — fall back to it for freshness.
+  }
+  epochs.push(Math.floor(statSync(path).mtimeMs / 1000));
+  return Math.max(...epochs);
+}
+
+/**
+ * Convenience for the session: COMPUTE the reconcile (pure) THEN PERSIST it (IO) in one
+ * call — the exact "reconcile → write → in-sync" step the `#sync-spec` eval leg performs.
+ * Still Model-B: the criteria DELTA (`request.proposed`) is the ai-architect's reasoning;
+ * this only applies it deterministically and writes it. Returns the compute result + the
+ * persist result together.
+ */
+export function reconcileAndPersistEvalCriteria(
+  request: EvalCriteriaReconcileRequest,
+  options: { cwd?: string; updatedAt?: string } = {},
+): { result: EvalCriteriaReconcileResult; persisted: PersistEvalCriteriaResult } {
+  const result = reconcileEvalCriteria(request);
+  const persisted = persistEvalCriteria(result, options);
+  return { result, persisted };
+}

package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts

@@ -27,7 +27,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { extractOutcomeSignals, type JudgeInvoke } from "./determine-outcome.ts";
-import { buildOutcomePrompt } from "./judge-prompt-template.ts";
+import { buildOutcomePrompt, type SubjectFrame } from "./judge-prompt-template.ts";
 import {
   writeJudgeTask,
   verdictFileName,
@@ -53,19 +53,27 @@ const PREP_PLACEHOLDER = JSON.stringify({
  */
 export function prepDeterminerTasks(
   traces: EvalTrace[],
-  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab },
+  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab; subject?: SubjectFrame },
 ): JudgeTaskSpec[] {
   return traces.map((trace) => {
+    // EV-1 — the subject frame MUST match what the Stage-B pipeline renders (byte-
+    // identity for the cross-stage cache), so the caller derives it from the SAME
+    // full trace batch buildSubjectProfile sees in run-pipeline.
     const { system, user } = buildOutcomePrompt(
       trace,
       extractOutcomeSignals(trace, opts.vocab),
       opts.vocab,
+      opts.subject,
     );
     return writeJudgeTask(opts.dir, {
       unit: { kind: "discover", traceId: trace.id },
       system,
       user,
       pin: opts.pin,
+      // INF-3 — salt the determiner cache key with the trace id so two sessions with
+      // an identical (e.g. empty) determiner prompt never collapse onto one verdict.
+      // AGGREGATE reads the stored `spec.verdictFile`, so this needs no re-derive there.
+      keyId: trace.id,
     });
   });
 }
@@ -84,8 +92,11 @@ export function createCapturingJudge(opts: {
   pin: PinnedEnvelope;
 }): { judge: JudgeInvoke; captured: JudgeTaskSpec[] } {
   const captured: JudgeTaskSpec[] = [];
-  const judge: JudgeInvoke = (system, user) => {
-    const vpath = join(opts.verdictDir, verdictFileName(system, user));
+  const judge: JudgeInvoke = (system, user, keyId) => {
+    // INF-3 — the determiner passes its trace id (keyId) so the salted stage-A
+    // verdict file is found; criterion-judge prompts pass no keyId ⇒ unsalted,
+    // and are CAPTURED as build-evals tasks below (byte-identical key, C-PIN).
+    const vpath = join(opts.verdictDir, verdictFileName(system, user, keyId));
     if (existsSync(vpath)) return Promise.resolve(readFileSync(vpath, "utf8"));
     captured.push(
       // D4 — the criterion judging axis (*build-evals / Step-2b). `kind` now matches the

package/.claude/skills/mutagent-evaluator/scripts/read-manifest.ts

@@ -0,0 +1,120 @@
+/**
+ * scripts/read-manifest.ts — EV-6: the sibling TraceManifest leg of the Discovery handoff.
+ * ---------------------------------------------------------------------------
+ * The Discovery handoff is a PATH PAIR — the UniTF `.jsonl` PLUS a sibling
+ * `TraceManifest` that reports the slice (count · format · truncation · coverage).
+ * Diagnostics reads BOTH and audits the slice BEFORE it judges; the evaluator used
+ * to read only the JSONL (a bare `--traces` path) and never opened the manifest —
+ * so a truncated or partially-malformed export was judged silently as if complete
+ * (EV-6). This wires the manifest leg into the evaluator intake, WARN-not-fail, the
+ * way diagnostics' `read-unitf.ts` `validateUniTFManifest` does.
+ *
+ * STANDALONE — the manifest shape is VENDORED (a minimal structural subset of
+ * `@mutagent/tools` `trace-manifest.ts`), NOT imported: the standalone-publish
+ * guard forbids the evaluator reaching into `@mutagent/tools`
+ * (MIGRATION-diagnostics-evaluator.md:417). Only the fields the audit reads are
+ * mirrored; diagnostics vendors its own identical copy.
+ *
+ * PURE except for the effectful `validateSiblingManifest` (one fs read of a sibling
+ * path). No clock / random / network.
+ */
+import { existsSync, readFileSync } from "node:fs";
+
+/** The manifest `format` marker that pairs with the UniTF version. */
+export const UNITF_MANIFEST_FORMAT = "unitf@0.1" as const;
+
+/**
+ * The sibling TraceManifest (frozen `TraceManifest v0.1.0`), structural subset —
+ * only the fields the slice audit inspects. Every field is optional so a partial
+ * or older manifest degrades to fewer warnings rather than throwing.
+ */
+export interface UnitfTraceManifest {
+  manifest_version?: string;
+  count?: number;
+  truncated?: boolean;
+  truncationReason?: string;
+  format?: string;
+  coverage?: {
+    fetched?: number;
+    malformed?: number;
+    deduped?: number;
+  };
+  warnings?: string[];
+}
+
+/**
+ * The sibling manifest path for a handed-over traces file: `<path>.manifest.json`,
+ * with a `.jsonl` suffix REPLACED (`traces.jsonl` → `traces.manifest.json`) —
+ * mirrors `@mutagent/tools` `manifestPathFor`. A `.gz` suffix is stripped first
+ * (the evaluator loads `.jsonl.gz` too) so `traces.jsonl.gz` also resolves to
+ * `traces.manifest.json`. PURE.
+ */
+export function manifestPathForTraces(tracesPath: string): string {
+  const base = tracesPath.endsWith(".gz") ? tracesPath.slice(0, -".gz".length) : tracesPath;
+  return base.endsWith(".jsonl")
+    ? base.slice(0, -".jsonl".length) + ".manifest.json"
+    : base + ".manifest.json";
+}
+
+/**
+ * Validate a parsed manifest against the parsed trace count. Returns WARNINGS
+ * (never throws, never aborts) — the migration's "surface warnings but do NOT
+ * abort a partial export" rule. Empty array = clean / no manifest. Mirrors
+ * diagnostics `validateUniTFManifest`.
+ */
+export function validateUnitfManifest(
+  manifest: UnitfTraceManifest | undefined,
+  actualTraceCount: number,
+): string[] {
+  const warnings: string[] = [];
+  if (!manifest) return warnings;
+  if (manifest.format !== undefined && manifest.format !== UNITF_MANIFEST_FORMAT) {
+    warnings.push(`manifest.format is "${manifest.format}" — expected "${UNITF_MANIFEST_FORMAT}"`);
+  }
+  if (manifest.count !== undefined && manifest.count !== actualTraceCount) {
+    warnings.push(
+      `manifest.count (${manifest.count}) != parsed trace count (${actualTraceCount}) — ` +
+        `the slice is incomplete (a truncated/partially-malformed export judged as if complete)`,
+    );
+  }
+  if (manifest.truncated === true) {
+    warnings.push(
+      `export is truncated${manifest.truncationReason ? ` (${manifest.truncationReason})` : ""} — ` +
+        `judging a PARTIAL slice`,
+    );
+  }
+  // Coverage cross-check: fetched should account for the parsed + malformed + deduped
+  // records; a fetched count that exceeds what landed on disk is a silent drop.
+  const cov = manifest.coverage;
+  if (cov?.fetched !== undefined) {
+    const accountedFor = actualTraceCount + (cov.malformed ?? 0) + (cov.deduped ?? 0);
+    if (cov.fetched !== accountedFor) {
+      warnings.push(
+        `manifest.coverage.fetched (${cov.fetched}) != parsed(${actualTraceCount}) + ` +
+          `malformed(${cov.malformed ?? 0}) + deduped(${cov.deduped ?? 0}) = ${accountedFor} — ` +
+          `records went missing between fetch and hand-off`,
+      );
+    }
+  }
+  if (manifest.warnings) warnings.push(...manifest.warnings);
+  return warnings;
+}
+
+/**
+ * Read + validate the sibling manifest for a handed-over traces file. Effectful
+ * (one fs read). NON-fatal throughout: an ABSENT manifest returns [] (the handoff
+ * may predate the manifest leg); an UNREADABLE / unparseable manifest yields a
+ * single warning and proceeds. Returns the human-readable warning list the caller
+ * surfaces on stderr.
+ */
+export function validateSiblingManifest(tracesPath: string, actualTraceCount: number): string[] {
+  const manifestPath = manifestPathForTraces(tracesPath);
+  if (!existsSync(manifestPath)) return []; // no manifest leg — nothing to audit
+  let manifest: UnitfTraceManifest | undefined;
+  try {
+    manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as UnitfTraceManifest;
+  } catch {
+    return [`sibling manifest '${manifestPath}' failed to read/parse — proceeding without slice audit`];
+  }
+  return validateUnitfManifest(manifest, actualTraceCount);
+}

package/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts

@@ -23,6 +23,30 @@ export interface ParsedUnitfTraces {
   traces: EvalTrace[];
   /** lines that failed to parse OR lacked the minimal UniTF shape (surfaced). */
   skipped: number;
+  /**
+   * INF-1 — traceIds of records that carry a prompt-bearing span (a user/system
+   * turn, or an assistant/agent GENERATION span) yet projected to an EMPTY
+   * `input.prompt`. This is the exact silent-empty symptom that read 21/21 blank
+   * on Claude Code traces: the reader must WARN, never emit empty in silence. The
+   * effectful reader/CLI surfaces this list; a non-empty list is a projection miss.
+   */
+  emptyPromptTraceIds: string[];
+}
+
+/**
+ * Does this record carry a span that SHOULD yield a prompt — an explicit user/
+ * system turn, or a GENERATION span (kind llm/agent, the Langfuse shape whose
+ * `input` holds the prompt)? Used to WARN when such a record still projects an
+ * empty `input.prompt` (the INF-1 silent-empty class). PURE.
+ */
+function hasPromptBearingSpan(rec: UnifiedTraceLike): boolean {
+  return rec.spans.some(
+    (s) =>
+      s.role === "user" ||
+      s.role === "system" ||
+      s.kind === "llm" ||
+      s.kind === "agent",
+  );
 }
 
 /**
@@ -45,6 +69,7 @@ function asUnitfRecord(value: unknown): UnifiedTraceLike | null {
  */
 export function parseUnitfJsonl(text: string): ParsedUnitfTraces {
   const traces: EvalTrace[] = [];
+  const emptyPromptTraceIds: string[] = [];
   let skipped = 0;
   for (const line of text.split("\n")) {
     const trimmed = line.trim();
@@ -61,9 +86,16 @@ export function parseUnitfJsonl(text: string): ParsedUnitfTraces {
       skipped += 1;
       continue;
     }
-    traces.push(projectUnitfToEvalTrace(rec));
+    const trace = projectUnitfToEvalTrace(rec);
+    traces.push(trace);
+    // INF-1 — WARN, never silently emit empty: a record with a prompt-bearing span
+    // that still projected an empty prompt is a projection miss (the Claude Code
+    // 21/21-blank class). Recorded here; the CLI/reader surfaces it.
+    if (hasPromptBearingSpan(rec) && (trace.input?.prompt ?? "").length === 0) {
+      emptyPromptTraceIds.push(trace.id);
+    }
   }
-  return { traces, skipped };
+  return { traces, skipped, emptyPromptTraceIds };
 }
 
 /**