@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts

@@ -27,7 +27,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { extractOutcomeSignals, type JudgeInvoke } from "./determine-outcome.ts";
-import { buildOutcomePrompt } from "./judge-prompt-template.ts";
+import { buildOutcomePrompt, type SubjectFrame } from "./judge-prompt-template.ts";
 import {
   writeJudgeTask,
   verdictFileName,
@@ -53,19 +53,27 @@ const PREP_PLACEHOLDER = JSON.stringify({
  */
 export function prepDeterminerTasks(
   traces: EvalTrace[],
-  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab },
+  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab; subject?: SubjectFrame },
 ): JudgeTaskSpec[] {
   return traces.map((trace) => {
+    // EV-1 — the subject frame MUST match what the Stage-B pipeline renders (byte-
+    // identity for the cross-stage cache), so the caller derives it from the SAME
+    // full trace batch buildSubjectProfile sees in run-pipeline.
     const { system, user } = buildOutcomePrompt(
       trace,
       extractOutcomeSignals(trace, opts.vocab),
       opts.vocab,
+      opts.subject,
     );
     return writeJudgeTask(opts.dir, {
       unit: { kind: "discover", traceId: trace.id },
       system,
       user,
       pin: opts.pin,
+      // INF-3 — salt the determiner cache key with the trace id so two sessions with
+      // an identical (e.g. empty) determiner prompt never collapse onto one verdict.
+      // AGGREGATE reads the stored `spec.verdictFile`, so this needs no re-derive there.
+      keyId: trace.id,
     });
   });
 }
@@ -84,8 +92,11 @@ export function createCapturingJudge(opts: {
   pin: PinnedEnvelope;
 }): { judge: JudgeInvoke; captured: JudgeTaskSpec[] } {
   const captured: JudgeTaskSpec[] = [];
-  const judge: JudgeInvoke = (system, user) => {
-    const vpath = join(opts.verdictDir, verdictFileName(system, user));
+  const judge: JudgeInvoke = (system, user, keyId) => {
+    // INF-3 — the determiner passes its trace id (keyId) so the salted stage-A
+    // verdict file is found; criterion-judge prompts pass no keyId ⇒ unsalted,
+    // and are CAPTURED as build-evals tasks below (byte-identical key, C-PIN).
+    const vpath = join(opts.verdictDir, verdictFileName(system, user, keyId));
     if (existsSync(vpath)) return Promise.resolve(readFileSync(vpath, "utf8"));
     captured.push(
       // D4 — the criterion judging axis (*build-evals / Step-2b). `kind` now matches the

package/.claude/skills/mutagent-evaluator/scripts/read-manifest.ts

@@ -0,0 +1,120 @@
+/**
+ * scripts/read-manifest.ts — EV-6: the sibling TraceManifest leg of the Discovery handoff.
+ * ---------------------------------------------------------------------------
+ * The Discovery handoff is a PATH PAIR — the UniTF `.jsonl` PLUS a sibling
+ * `TraceManifest` that reports the slice (count · format · truncation · coverage).
+ * Diagnostics reads BOTH and audits the slice BEFORE it judges; the evaluator used
+ * to read only the JSONL (a bare `--traces` path) and never opened the manifest —
+ * so a truncated or partially-malformed export was judged silently as if complete
+ * (EV-6). This wires the manifest leg into the evaluator intake, WARN-not-fail, the
+ * way diagnostics' `read-unitf.ts` `validateUniTFManifest` does.
+ *
+ * STANDALONE — the manifest shape is VENDORED (a minimal structural subset of
+ * `@mutagent/tools` `trace-manifest.ts`), NOT imported: the standalone-publish
+ * guard forbids the evaluator reaching into `@mutagent/tools`
+ * (MIGRATION-diagnostics-evaluator.md:417). Only the fields the audit reads are
+ * mirrored; diagnostics vendors its own identical copy.
+ *
+ * PURE except for the effectful `validateSiblingManifest` (one fs read of a sibling
+ * path). No clock / random / network.
+ */
+import { existsSync, readFileSync } from "node:fs";
+
+/** The manifest `format` marker that pairs with the UniTF version. */
+export const UNITF_MANIFEST_FORMAT = "unitf@0.1" as const;
+
+/**
+ * The sibling TraceManifest (frozen `TraceManifest v0.1.0`), structural subset —
+ * only the fields the slice audit inspects. Every field is optional so a partial
+ * or older manifest degrades to fewer warnings rather than throwing.
+ */
+export interface UnitfTraceManifest {
+  manifest_version?: string;
+  count?: number;
+  truncated?: boolean;
+  truncationReason?: string;
+  format?: string;
+  coverage?: {
+    fetched?: number;
+    malformed?: number;
+    deduped?: number;
+  };
+  warnings?: string[];
+}
+
+/**
+ * The sibling manifest path for a handed-over traces file: `<path>.manifest.json`,
+ * with a `.jsonl` suffix REPLACED (`traces.jsonl` → `traces.manifest.json`) —
+ * mirrors `@mutagent/tools` `manifestPathFor`. A `.gz` suffix is stripped first
+ * (the evaluator loads `.jsonl.gz` too) so `traces.jsonl.gz` also resolves to
+ * `traces.manifest.json`. PURE.
+ */
+export function manifestPathForTraces(tracesPath: string): string {
+  const base = tracesPath.endsWith(".gz") ? tracesPath.slice(0, -".gz".length) : tracesPath;
+  return base.endsWith(".jsonl")
+    ? base.slice(0, -".jsonl".length) + ".manifest.json"
+    : base + ".manifest.json";
+}
+
+/**
+ * Validate a parsed manifest against the parsed trace count. Returns WARNINGS
+ * (never throws, never aborts) — the migration's "surface warnings but do NOT
+ * abort a partial export" rule. Empty array = clean / no manifest. Mirrors
+ * diagnostics `validateUniTFManifest`.
+ */
+export function validateUnitfManifest(
+  manifest: UnitfTraceManifest | undefined,
+  actualTraceCount: number,
+): string[] {
+  const warnings: string[] = [];
+  if (!manifest) return warnings;
+  if (manifest.format !== undefined && manifest.format !== UNITF_MANIFEST_FORMAT) {
+    warnings.push(`manifest.format is "${manifest.format}" — expected "${UNITF_MANIFEST_FORMAT}"`);
+  }
+  if (manifest.count !== undefined && manifest.count !== actualTraceCount) {
+    warnings.push(
+      `manifest.count (${manifest.count}) != parsed trace count (${actualTraceCount}) — ` +
+        `the slice is incomplete (a truncated/partially-malformed export judged as if complete)`,
+    );
+  }
+  if (manifest.truncated === true) {
+    warnings.push(
+      `export is truncated${manifest.truncationReason ? ` (${manifest.truncationReason})` : ""} — ` +
+        `judging a PARTIAL slice`,
+    );
+  }
+  // Coverage cross-check: fetched should account for the parsed + malformed + deduped
+  // records; a fetched count that exceeds what landed on disk is a silent drop.
+  const cov = manifest.coverage;
+  if (cov?.fetched !== undefined) {
+    const accountedFor = actualTraceCount + (cov.malformed ?? 0) + (cov.deduped ?? 0);
+    if (cov.fetched !== accountedFor) {
+      warnings.push(
+        `manifest.coverage.fetched (${cov.fetched}) != parsed(${actualTraceCount}) + ` +
+          `malformed(${cov.malformed ?? 0}) + deduped(${cov.deduped ?? 0}) = ${accountedFor} — ` +
+          `records went missing between fetch and hand-off`,
+      );
+    }
+  }
+  if (manifest.warnings) warnings.push(...manifest.warnings);
+  return warnings;
+}
+
+/**
+ * Read + validate the sibling manifest for a handed-over traces file. Effectful
+ * (one fs read). NON-fatal throughout: an ABSENT manifest returns [] (the handoff
+ * may predate the manifest leg); an UNREADABLE / unparseable manifest yields a
+ * single warning and proceeds. Returns the human-readable warning list the caller
+ * surfaces on stderr.
+ */
+export function validateSiblingManifest(tracesPath: string, actualTraceCount: number): string[] {
+  const manifestPath = manifestPathForTraces(tracesPath);
+  if (!existsSync(manifestPath)) return []; // no manifest leg — nothing to audit
+  let manifest: UnitfTraceManifest | undefined;
+  try {
+    manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as UnitfTraceManifest;
+  } catch {
+    return [`sibling manifest '${manifestPath}' failed to read/parse — proceeding without slice audit`];
+  }
+  return validateUnitfManifest(manifest, actualTraceCount);
+}

package/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts

@@ -23,6 +23,30 @@ export interface ParsedUnitfTraces {
   traces: EvalTrace[];
   /** lines that failed to parse OR lacked the minimal UniTF shape (surfaced). */
   skipped: number;
+  /**
+   * INF-1 — traceIds of records that carry a prompt-bearing span (a user/system
+   * turn, or an assistant/agent GENERATION span) yet projected to an EMPTY
+   * `input.prompt`. This is the exact silent-empty symptom that read 21/21 blank
+   * on Claude Code traces: the reader must WARN, never emit empty in silence. The
+   * effectful reader/CLI surfaces this list; a non-empty list is a projection miss.
+   */
+  emptyPromptTraceIds: string[];
+}
+
+/**
+ * Does this record carry a span that SHOULD yield a prompt — an explicit user/
+ * system turn, or a GENERATION span (kind llm/agent, the Langfuse shape whose
+ * `input` holds the prompt)? Used to WARN when such a record still projects an
+ * empty `input.prompt` (the INF-1 silent-empty class). PURE.
+ */
+function hasPromptBearingSpan(rec: UnifiedTraceLike): boolean {
+  return rec.spans.some(
+    (s) =>
+      s.role === "user" ||
+      s.role === "system" ||
+      s.kind === "llm" ||
+      s.kind === "agent",
+  );
 }
 
 /**
@@ -45,6 +69,7 @@ function asUnitfRecord(value: unknown): UnifiedTraceLike | null {
  */
 export function parseUnitfJsonl(text: string): ParsedUnitfTraces {
   const traces: EvalTrace[] = [];
+  const emptyPromptTraceIds: string[] = [];
   let skipped = 0;
   for (const line of text.split("\n")) {
     const trimmed = line.trim();
@@ -61,9 +86,16 @@ export function parseUnitfJsonl(text: string): ParsedUnitfTraces {
       skipped += 1;
       continue;
     }
-    traces.push(projectUnitfToEvalTrace(rec));
+    const trace = projectUnitfToEvalTrace(rec);
+    traces.push(trace);
+    // INF-1 — WARN, never silently emit empty: a record with a prompt-bearing span
+    // that still projected an empty prompt is a projection miss (the Claude Code
+    // 21/21-blank class). Recorded here; the CLI/reader surfaces it.
+    if (hasPromptBearingSpan(rec) && (trace.input?.prompt ?? "").length === 0) {
+      emptyPromptTraceIds.push(trace.id);
+    }
   }
-  return { traces, skipped };
+  return { traces, skipped, emptyPromptTraceIds };
 }
 
 /**

package/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts

@@ -83,6 +83,33 @@ export function renderBuildEvalsProgressCard(p: BuildEvalsProgress): string {
   ].join("\n");
 }
 
+// ── EVAL-1 — "what this stage will produce" notice (up-front, at stage start) ─
+export interface StageProducesNotice {
+  subject: string;
+  /** the command about to run (e.g. "*build-dataset" | "*build-evals" | "*eval"). */
+  command: string;
+  /** the named deliverables this stage emits (dataset · criteria · which files). */
+  produces: string[];
+}
+
+/**
+ * EVAL-1 (1) — render the up-front notice that names WHAT this stage will produce,
+ * so a build never runs silently with the operator unaware a dataset/criteria are
+ * coming. Emitted at Step 0, BEFORE the build. PURE. NOT an approval gate — a
+ * notification only (operator constraint).
+ */
+export function renderStageProducesNotice(n: StageProducesNotice): string {
+  const lines = [
+    TOP,
+    row(`▷ ${n.command} — ${n.subject}`),
+    MID,
+    row("this stage will produce:"),
+  ];
+  for (const p of n.produces) lines.push(row(`  • ${p}`));
+  lines.push(BOT);
+  return lines.join("\n");
+}
+
 // ── F22 — dataset entity card (verbose, after *build-dataset) ─────────────────
 export interface DatasetEntity {
   subject: string;
@@ -99,6 +126,12 @@ export function renderDatasetEntityCard(e: DatasetEntity): string {
     MID,
     row(`categories (${e.categories.length}) · ${e.totalItems} items total`),
   ];
+  // EVAL-1 — a build that produced NOTHING must WARN, never render a blank card that
+  // reads as a silent success (the dogfood symptom: a build that reported and a build
+  // that stayed silent looked identical).
+  if (e.categories.length === 0 || e.totalItems === 0) {
+    lines.push(row("⚠ EMPTY — this build produced no dataset items (nothing to evaluate)."));
+  }
   for (const c of e.categories) {
     lines.push(row(`  • ${c.id}: ${c.items} items  (edge: ${c.edgeItems})`));
   }
@@ -129,6 +162,10 @@ export function renderEvalsEntityCard(e: EvalsEntity): string {
     row(`output sink: ${e.outputSink}`),
     row(`criteria (${e.criteria.length}):`),
   ];
+  // EVAL-1 — an eval suite with zero criteria is a silent no-op build; WARN loudly.
+  if (e.criteria.length === 0) {
+    lines.push(row("⚠ EMPTY — this build produced no criteria (the suite judges nothing)."));
+  }
   for (const c of e.criteria) {
     lines.push(row(`  • ${c.id}  [${c.severity}·${c.kind}]`));
   }