@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts

@@ -0,0 +1,125 @@
+/**
+ * scripts/prep-tasks.ts — the deterministic PREP half of the agent-dispatch engine.
+ * ---------------------------------------------------------------------------
+ * The agent-dispatch substrate splits judging into PREP (this file) + DISPATCH
+ * (the parent session, per references/workflows/orchestrator-protocol.md) +
+ * AGGREGATE (run-pipeline.ts reading the verdict files). PREP emits one
+ * task-spec file per judging unit — the EXACT (system, user) prompt a dispatched
+ * leaf subagent must reason over, keyed by a content-hash so AGGREGATE re-derives
+ * the same key from the prompt alone.
+ *
+ * There is a HARD STAGE BARRIER (the same dependency diagnostics has between its
+ * deep-read and its analyzer fan-out): the determiner labels (EV-042) must be
+ * REAL before the *build-evals judge prompts can be built, because the judge
+ * few-shot block is drawn from the determiner-labeled TRAIN split. So PREP runs
+ * in two stages:
+ *
+ *   Stage A — prepDeterminerTasks  (no label dependency; pure per-trace prompt).
+ *             → dispatch error-analyst → collect determiner verdict files.
+ *   Stage B — prepJudgeTasks       (REQUIRES stage-A verdicts collected; replays
+ *             the pipeline with a capturing judge so the emitted judge prompts
+ *             are byte-identical to what AGGREGATE will build).
+ *
+ * PURE except for fs (writes task-spec files; reads stage-A verdict files). No
+ * clock / random / network. Model intent SACRED: the pinned envelope is carried
+ * on every task spec for the host runtime to honor.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { extractOutcomeSignals, type JudgeInvoke } from "./determine-outcome.ts";
+import { buildOutcomePrompt } from "./judge-prompt-template.ts";
+import {
+  writeJudgeTask,
+  verdictFileName,
+  type JudgeTaskSpec,
+  type PinnedEnvelope,
+} from "./agent-dispatch.ts";
+import { runEvalPipeline } from "./run-pipeline.ts";
+import type { PipelineOptions } from "./run-pipeline.ts";
+import type { EvalTrace, SubjectVocab } from "./contracts/eval-types.ts";
+
+/** A PREP placeholder verdict — lets the pipeline complete during stage-B capture; never reported. */
+const PREP_PLACEHOLDER = JSON.stringify({
+  critique: "PREP placeholder — awaiting the dispatched subagent's verdict file",
+  result: "pass",
+  confidence: 0,
+});
+
+/**
+ * Stage A PREP — emit one determiner (EV-042) task-spec per trace. The
+ * determiner prompt has NO label dependency (it reads the trace's own event +
+ * trajectory + terminal state), so these can be emitted + dispatched first.
+ * The dispatched error-analyst writes each verdict file; AGGREGATE-A reads them.
+ */
+export function prepDeterminerTasks(
+  traces: EvalTrace[],
+  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab },
+): JudgeTaskSpec[] {
+  return traces.map((trace) => {
+    const { system, user } = buildOutcomePrompt(
+      trace,
+      extractOutcomeSignals(trace, opts.vocab),
+      opts.vocab,
+    );
+    return writeJudgeTask(opts.dir, {
+      unit: { kind: "discover", traceId: trace.id },
+      system,
+      user,
+      pin: opts.pin,
+    });
+  });
+}
+
+/**
+ * A capturing JudgeInvoke: when a prompt's verdict file is already present
+ * (stage-A determiner verdicts), READ it; otherwise CAPTURE the prompt as a
+ * judge task-spec and return a deterministic placeholder so the pipeline runs to
+ * completion. Replaying the real pipeline with this judge emits the *build-evals
+ * judge prompts using the REAL determiner labels — guaranteeing they are
+ * byte-identical to the prompts AGGREGATE will build (same code, same labels).
+ */
+export function createCapturingJudge(opts: {
+  verdictDir: string;
+  taskDir: string;
+  pin: PinnedEnvelope;
+}): { judge: JudgeInvoke; captured: JudgeTaskSpec[] } {
+  const captured: JudgeTaskSpec[] = [];
+  const judge: JudgeInvoke = (system, user) => {
+    const vpath = join(opts.verdictDir, verdictFileName(system, user));
+    if (existsSync(vpath)) return Promise.resolve(readFileSync(vpath, "utf8"));
+    captured.push(
+      // D4 — the criterion judging axis (*build-evals / Step-2b). `kind` now matches the
+      // command (was mis-tagged "evaluate" pre-merge). ENVELOPE-ONLY retag: `unit` is not
+      // part of promptHash(system, user), so the rendered prompt + its key (8b58d9ca) are
+      // invariant — only the task.json envelope carries the new {kind, axis}.
+      writeJudgeTask(opts.taskDir, {
+        unit: { kind: "build-evals", axis: "criterion" },
+        system,
+        user,
+        pin: opts.pin,
+      }),
+    );
+    return Promise.resolve(PREP_PLACEHOLDER);
+  };
+  return { judge, captured };
+}
+
+/**
+ * Stage B PREP — emit the *build-evals / *evaluate judge (EV-043/048) task-specs.
+ * REQUIRES the stage-A determiner verdict files to already be collected in
+ * `verdictDir` (else the determiner prompts would themselves be captured with
+ * placeholder labels, corrupting the judge few-shot). Replays runEvalPipeline
+ * with the capturing judge and returns the captured judge tasks for dispatch.
+ */
+export async function prepJudgeTasks(
+  traces: EvalTrace[],
+  opts: { verdictDir: string; taskDir: string; pin: PinnedEnvelope; pipeline: PipelineOptions },
+): Promise<JudgeTaskSpec[]> {
+  const { judge, captured } = createCapturingJudge({
+    verdictDir: opts.verdictDir,
+    taskDir: opts.taskDir,
+    pin: opts.pin,
+  });
+  await runEvalPipeline(traces, judge, opts.pipeline);
+  return captured;
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/profile-subject.ts

@@ -0,0 +1,310 @@
+/**
+ * scripts/profile-subject.ts — EV-049 subject auto-gen (NEVER hand-authored).
+ * ---------------------------------------------------------------------------
+ * Generate a subject profile from a trace sample (code/platform/trace
+ * exploration). For sample this is the EV-049 "scroll the traces, infer the
+ * invoked tools" path: the 35-tool inventory falls out of
+ * observations[].type=="TOOL" (sample-findings §1). Also infers the event-kind
+ * taxonomy, the observation-count clusters, and a platform marker.
+ *
+ * This is the PURE trace-exploration core (a distinct file from the v1
+ * cli/profile-subject.ts, which profiles a skill/agent from its source for the
+ * `*audit` surface). PURE + deterministic: tools ranked by frequency then name;
+ * no clock / random / network.
+ */
+import {
+  classifyEvent,
+} from "./determine-outcome.ts";
+import {
+  UNCLASSIFIED_EVENT,
+  parseSubjectVocab,
+  type EvalTrace,
+  type EventTagRule,
+  type SubjectVocab,
+} from "./contracts/eval-types.ts";
+import { buildFlowGraph } from "./flow-graph.ts";
+import {
+  parseExpectedFlow,
+  type ExpectedFlow,
+  type ExpectedFlowEdge,
+  type FlowGraph,
+} from "./contracts/flow-graph.ts";
+
+export interface ToolStat {
+  name: string;
+  count: number;
+}
+
+export interface SubjectProfile {
+  subjectName: string;
+  traceCount: number;
+  toolInventory: ToolStat[];
+  /**
+   * The SUBJECT VOCABULARY the engine reads (EV-002 / EV-049). Either AUTHORED
+   * (passed to `profileSubject`) or best-effort auto-generated from the traces.
+   * The determiner reads its tag→kind rules / tool names from HERE, never from a
+   * module constant.
+   */
+  vocab: SubjectVocab;
+  /** event-kind distribution over the sample (counts), keyed by the vocab kinds. */
+  eventTaxonomy: Record<string, number>;
+  /** observation-count -> number of traces with that count (discrete clusters). */
+  obsCountClusters: Record<number, number>;
+  platform: string;
+  /**
+   * The agent-shaped EXPECTED-FLOW spec (EV-037) the context-flow audit diffs
+   * against. AUTHORED when supplied (validated) — the canonical path for
+   * fidelity — else best-effort auto-generated from the traces (EV-049): the
+   * modal producer→consumer threadings the subject normally exhibits. The
+   * semantic fields the trace alone can't carry (which tools DISPATCH sub-agents
+   * · which slots the subject's HTML SHOULD render) are left empty with a TODO,
+   * exactly like the vocab's recovery/send/guard fields.
+   */
+  expectedFlow: ExpectedFlow;
+}
+
+/** Tool inventory ranked by frequency desc, then name asc (deterministic). */
+export function inferToolInventory(traces: EvalTrace[]): ToolStat[] {
+  const counts = new Map<string, number>();
+  for (const t of traces) {
+    for (const o of t.observations) {
+      if (o.type === "TOOL" && typeof o.name === "string") {
+        counts.set(o.name, (counts.get(o.name) ?? 0) + 1);
+      }
+    }
+  }
+  return [...counts.entries()]
+    .map(([name, count]) => ({ name, count }))
+    .sort((a, b) => (b.count - a.count !== 0 ? b.count - a.count : a.name.localeCompare(b.name)));
+}
+
+function promptOf(trace: EvalTrace): string {
+  return typeof trace.input?.prompt === "string" ? trace.input.prompt : "";
+}
+
+/**
+ * Pull a single message's textual content out of the loosely-typed payload. The
+ * AI-SDK / Langfuse shape carries `content` either as a plain string OR as an
+ * array of content parts (`{type:"text", text}` / `{text}`). Returns the joined
+ * text, or "" when nothing textual is present. PURE — read-only narrowing.
+ */
+function messageContentText(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content
+      .map((part) =>
+        typeof part === "string"
+          ? part
+          : part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
+            ? (part as { text: string }).text
+            : "",
+      )
+      .join("");
+  }
+  return "";
+}
+
+/**
+ * Coerce an observation `input` to a message list. The AI-SDK GENERATION span
+ * carries its prompt EITHER as a bare message array (`[{role,content},…]`) OR as
+ * `{messages:[…]}`. Returns the message array, or `[]` when the input is neither
+ * shape. PURE — read-only narrowing over the `unknown` payload.
+ */
+function messagesOf(input: unknown): unknown[] {
+  if (Array.isArray(input)) return input;
+  if (input && typeof input === "object") {
+    const m = (input as { messages?: unknown }).messages;
+    if (Array.isArray(m)) return m;
+  }
+  return [];
+}
+
+/**
+ * RECONSTRUCT the subject's SYSTEM PROMPT from the trace batch (UI-14). The agent
+ * never hands us its prompt in the trace-only path, but every LLM call carries it:
+ * each GENERATION observation's `input` is the message list sent to the model, and
+ * its first `role:"system"` message IS the system prompt. We scan observations in
+ * order and return the FIRST non-empty system-message content found across the
+ * batch (deterministic: trace order, then observation order).
+ *
+ * Handles the langfuse / AI-SDK shapes: `input` as a bare message array OR
+ * `{messages:[…]}`; the system `content` as a string OR an array of text parts.
+ *
+ * NEVER confabulates: when no observation carries a `role:"system"` message the
+ * function returns `undefined` (the caller keeps the prompt UNAVAILABLE). PURE —
+ * no clock / random / network.
+ */
+export function inferSystemPrompt(traces: EvalTrace[]): string | undefined {
+  for (const t of traces) {
+    for (const o of t.observations) {
+      for (const msg of messagesOf(o.input)) {
+        if (msg && typeof msg === "object" && (msg as { role?: unknown }).role === "system") {
+          const text = messageContentText((msg as { content?: unknown }).content).trim();
+          if (text.length > 0) return text;
+        }
+      }
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Auto-infer the subject's event-tag rules from the traces (EV-049,
+ * subject-agnostic extraction). Scans each prompt for opening tags `<{name}` and
+ * collects the distinct tag names, ranked by per-trace presence (desc) then name
+ * (asc). Tags present in EVERY trace are treated as ubiquitous WRAPPERS (e.g. a
+ * `<current_time>` envelope) and dropped — they don't partition the sample into
+ * event kinds. Each surviving tag becomes a `{kind: tag, tag}` rule.
+ *
+ * TODO(EV-049): stem-collapse (e.g. `opportunity_update` → `opportunity`) needs
+ * subject semantics the trace alone doesn't carry — supply an AUTHORED vocab
+ * (profileSubject's 2nd arg) when canonical kind names matter.
+ */
+export function inferEventTags(traces: EvalTrace[]): EventTagRule[] {
+  const presence = new Map<string, number>();
+  for (const t of traces) {
+    const prompt = promptOf(t);
+    const seen = new Set<string>();
+    for (const m of prompt.matchAll(/<([a-zA-Z][\w-]*)/g)) seen.add(m[1]);
+    for (const tag of seen) presence.set(tag, (presence.get(tag) ?? 0) + 1);
+  }
+  const total = traces.length;
+  return [...presence.entries()]
+    .filter(([, n]) => total <= 1 || n < total) // drop ubiquitous wrappers
+    .sort((a, b) => (b[1] - a[1] !== 0 ? b[1] - a[1] : a[0].localeCompare(b[0])))
+    .map(([tag]) => ({ kind: tag, tag }));
+}
+
+/**
+ * Best-effort auto-generated subject vocab when no AUTHORED vocab is supplied.
+ * Event tags are inferred from the traces; the SEMANTIC fields (which tools are
+ * "recovery" / which tool is the "send" action / the guard counter attribute)
+ * are left empty with a TODO — they need subject semantics the trace alone
+ * doesn't carry, so they are supplied via an authored vocab until that inference
+ * lands. The ENGINE reads the vocab off the profile either way (EV-002).
+ */
+function inferVocab(traces: EvalTrace[]): SubjectVocab {
+  return {
+    recoveryTools: [], // TODO(EV-049): infer recovery-tool semantics from traces.
+    eventTags: inferEventTags(traces),
+    sendTool: "", // TODO(EV-049): infer the primary send-action tool from traces.
+    guardCounterAttr: null, // TODO(EV-049): infer the guard counter attribute.
+  };
+}
+
+/** Event-kind distribution over the sample, using the subject's tag rules. */
+export function inferEventTaxonomy(
+  traces: EvalTrace[],
+  vocab: SubjectVocab,
+): Record<string, number> {
+  const tax: Record<string, number> = { [UNCLASSIFIED_EVENT]: 0 };
+  for (const rule of vocab.eventTags) tax[rule.kind] = 0;
+  for (const t of traces) {
+    const kind = classifyEvent(promptOf(t), vocab);
+    tax[kind] = (tax[kind] ?? 0) + 1;
+  }
+  return tax;
+}
+
+/** Name-keyed expected-flow edges from one graph's THREADED data-handoffs. */
+export function expectedEdgesFromGraph(graph: FlowGraph): ExpectedFlowEdge[] {
+  const nameById = new Map(graph.nodes.map((n) => [n.id, n.name]));
+  const out: ExpectedFlowEdge[] = [];
+  for (const e of graph.edges) {
+    const fromTool = nameById.get(e.from) ?? "";
+    const toTool = nameById.get(e.to) ?? "";
+    if (fromTool.length === 0 || toTool.length === 0) continue; // unnamed → not normative
+    out.push({ fromTool, toTool, slot: e.slot });
+  }
+  return out;
+}
+
+/**
+ * Auto-infer the subject's EXPECTED-FLOW from the trace sample (EV-049,
+ * subject-agnostic). The expected threadings = the UNION of producer→consumer
+ * data-handoffs actually observed across the sample (deduped, deterministically
+ * ordered) — the "normal" flow that SHOULD keep happening (a regression
+ * baseline). The semantic fields the trace cannot carry are left empty with a
+ * TODO: `dispatchToolNames` (which tools dispatch sub-agents) and
+ * `expectedUiSlots` (which slots the subject's HTML artifact SHOULD render) need
+ * subject semantics — supply an AUTHORED expected-flow when they matter.
+ */
+export function inferExpectedFlow(traces: EvalTrace[]): ExpectedFlow {
+  const seen = new Map<string, ExpectedFlowEdge>();
+  for (const t of traces) {
+    const graph = buildFlowGraph(t);
+    for (const edge of expectedEdgesFromGraph(graph)) {
+      const key = `${edge.fromTool}␟${edge.toTool}␟${edge.slot ?? ""}`;
+      if (!seen.has(key)) seen.set(key, edge);
+    }
+  }
+  const edges = [...seen.values()].sort((a, b) =>
+    a.fromTool !== b.fromTool
+      ? a.fromTool.localeCompare(b.fromTool)
+      : a.toTool !== b.toTool
+        ? a.toTool.localeCompare(b.toTool)
+        : (a.slot ?? "").localeCompare(b.slot ?? ""),
+  );
+  return {
+    dispatchToolNames: [], // TODO(EV-037): infer sub-agent dispatch semantics; authored until then.
+    edges,
+    expectedUiSlots: [], // TODO(EV-037): the HTML render slots aren't in agent traces; authored.
+  };
+}
+
+/** Discrete observation-count clusters: obsCount -> #traces (sample's 4/14/28/44). */
+function inferObsClusters(traces: EvalTrace[]): Record<number, number> {
+  const clusters: Record<number, number> = {};
+  for (const t of traces) {
+    const n = t.observations.length;
+    clusters[n] = (clusters[n] ?? 0) + 1;
+  }
+  return clusters;
+}
+
+/** Platform marker from generation-span names (closed, explicit — no fuzzing). */
+export function inferPlatform(traces: EvalTrace[]): string {
+  for (const t of traces) {
+    for (const o of t.observations) {
+      const name = typeof o.name === "string" ? o.name : "";
+      if (/^ai\.generateText\b/.test(name) || /^agent\.step\./.test(name)) {
+        return "vercel-ai-sdk";
+      }
+    }
+  }
+  return "unknown";
+}
+
+/** The subject name = the (uniform) trace.name, else "unknown-subject". */
+function inferSubjectName(traces: EvalTrace[]): string {
+  const first = traces.find((t) => typeof t.name === "string" && t.name.length > 0);
+  return first?.name ?? "unknown-subject";
+}
+
+/**
+ * Assemble the full subject profile. PURE + deterministic. The subject VOCAB is
+ * AUTHORED when supplied (validated against the schema) — the canonical path for
+ * fidelity — else best-effort auto-inferred from the traces (EV-049). The engine
+ * reads its subject vocabulary off `profile.vocab` (EV-002), never a module
+ * constant.
+ */
+export function profileSubject(
+  traces: EvalTrace[],
+  authoredVocab?: SubjectVocab,
+  authoredFlow?: ExpectedFlow,
+): SubjectProfile {
+  const vocab =
+    authoredVocab !== undefined ? parseSubjectVocab(authoredVocab) : inferVocab(traces);
+  const expectedFlow =
+    authoredFlow !== undefined ? parseExpectedFlow(authoredFlow) : inferExpectedFlow(traces);
+  return {
+    subjectName: inferSubjectName(traces),
+    traceCount: traces.length,
+    toolInventory: inferToolInventory(traces),
+    vocab,
+    eventTaxonomy: inferEventTaxonomy(traces, vocab),
+    obsCountClusters: inferObsClusters(traces),
+    platform: inferPlatform(traces),
+    expectedFlow,
+  };
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/publish-report.ts

@@ -0,0 +1,131 @@
+/**
+ * scripts/publish-report.ts — UI-13: the publish-time STRIPPER (client-safe HTML).
+ * ---------------------------------------------------------------------------
+ * The eval-report (`render-eval-report.ts`) is INTERNAL-grade: it carries a §5
+ * Self-Eval [INTERNAL] calibration panel + `data-strip="strip-for-client"` markers
+ * + raw triggering INPUT (§2) and verbatim EVIDENCE (§4) that may contain PII when
+ * the source batch was NOT pre-sanitized. Those markers were declarative ONLY — no
+ * code enforced them. This module enforces them:
+ *
+ *   stripForClient(html, opts) →
+ *     ALWAYS removes  · the §5 internal panel (`<section id="t5">…`)
+ *                     · the internal tab-button (`.tab-btn.internal`)
+ *                     · every `data-strip="strip-for-client"` node
+ *     PII gate (unless opts.assumeSanitized) →
+ *                     · BLANKS the raw triggering inputs embedded in the §2 ledger
+ *                       data blob (`data-pii="ledger"` script: `"input":"…"` → "")
+ *                     · REDACTS the §4 verbatim-evidence regions (`data-pii="evidence"`)
+ *
+ * The result is a CLIENT-SAFE HTML string: no §5/INTERNAL content, and (by default)
+ * no raw PII dumped onto the public tabs. Pass `{ assumeSanitized: true }` (CLI:
+ * `--assume-sanitized`) ONLY for a batch you have CONFIRMED carries no PII — then the
+ * raw input + verbatim evidence are preserved.
+ *
+ * PURE + deterministic: a string→string transform, no clock/random, no I/O (the CLI
+ * wrapper at the bottom does the read/write). Idempotent: stripping an already-
+ * stripped HTML is a no-op (the markers are gone, the regexes match nothing).
+ */
+
+export interface StripOptions {
+  /** When TRUE, the batch is asserted PII-free: raw input + verbatim evidence are
+   *  PRESERVED (only the §5 internal panel is stripped). DEFAULT false — the PII
+   *  gate redacts raw input + evidence so an un-sanitized batch never leaks. */
+  assumeSanitized?: boolean;
+}
+
+/** The placeholder substituted for a redacted PII region (publish, un-sanitized). */
+export const PII_REDACTION =
+  "⟨redacted on publish — raw content withheld; re-run with --assume-sanitized for a confirmed-PII-free batch⟩";
+
+/**
+ * Strip the INTERNAL surfaces (and, unless `assumeSanitized`, the raw PII) from a
+ * rendered eval-report → a client-safe HTML string. See the module header.
+ */
+export function stripForClient(html: string, opts: StripOptions = {}): string {
+  let out = html;
+
+  // 1) ALWAYS — remove the §5 Self-Eval [INTERNAL] panel (whole <section id="t5">).
+  out = out.replace(/<section\b[^>]*\bid="t5"[^>]*>[\s\S]*?<\/section>/g, "");
+
+  // 2) ALWAYS — remove the internal tab-button (.tab-btn.internal → data-tab="t5").
+  out = out.replace(/<button\b[^>]*\bclass="tab-btn internal"[^>]*>[\s\S]*?<\/button>/g, "");
+
+  // 3) ALWAYS — remove every node explicitly marked `data-strip="strip-for-client"`
+  //    (tag-agnostic; matches the element + its content). Any leftover after (1).
+  out = out.replace(
+    /<(\w+)\b[^>]*\bdata-strip="strip-for-client"[^>]*>[\s\S]*?<\/\1>/g,
+    "",
+  );
+
+  // 4) PII GATE — unless the batch is asserted sanitized, scrub the raw PII.
+  if (opts.assumeSanitized !== true) {
+    // 4a) BLANK the raw triggering INPUTs embedded in the §2 ledger data blob. The
+    //     blob is the `data-pii="ledger"` <script>; within it, every `"input":"…"`
+    //     JSON value (the verbatim user/agent prompt) is emptied. The JSON-string
+    //     pattern `"(?:[^"\\]|\\.)*"` correctly spans escaped quotes.
+    out = out.replace(
+      /(<script\b[^>]*\bdata-pii="ledger"[^>]*>)([\s\S]*?)(<\/script>)/g,
+      (_m, open: string, body: string, close: string) =>
+        open + body.replace(/"input":"(?:[^"\\]|\\.)*"/g, '"input":""') + close,
+    );
+
+    // 4b) REDACT the §4 verbatim-evidence regions (`data-pii="evidence"`): replace
+    //     the element's INNER content with the redaction placeholder (keep the tags
+    //     so the layout is intact). No nested same-tag inside these cells.
+    out = out.replace(
+      /(<(\w+)\b[^>]*\bdata-pii="evidence"[^>]*>)[\s\S]*?(<\/\2>)/g,
+      (_m, open: string, _tag: string, close: string) => open + PII_REDACTION + close,
+    );
+  }
+
+  return out;
+}
+
+// ── CLI entrypoint ──────────────────────────────────────────────────────────
+//
+// bun scripts/publish-report.ts <report.html> [out.client.html] [--assume-sanitized]
+// Reads a rendered eval-report, strips the INTERNAL surfaces (+ PII unless the flag),
+// and writes the client-safe HTML. Default out = `<input>.client.html`.
+
+declare const Bun: { argv: string[] } | undefined;
+
+async function main(): Promise<void> {
+  const argv = typeof Bun !== "undefined" ? Bun.argv.slice(2) : process.argv.slice(2);
+  const flags = new Set(argv.filter((a) => a.startsWith("--")));
+  const positional = argv.filter((a) => !a.startsWith("--"));
+  const [inputPath, outArg] = positional;
+  if (!inputPath) {
+    console.error(
+      "usage: publish-report.ts <report.html> [out.client.html] [--assume-sanitized]",
+    );
+    process.exit(2);
+    return;
+  }
+  const { readFileSync, writeFileSync } = await import("node:fs");
+  const assumeSanitized = flags.has("--assume-sanitized");
+  const html = readFileSync(inputPath, "utf8");
+  const client = stripForClient(html, { assumeSanitized });
+  const outPath = outArg ?? inputPath.replace(/\.html$/i, "") + ".client.html";
+  writeFileSync(outPath, client);
+  console.info(
+    JSON.stringify(
+      {
+        input: inputPath,
+        output: outPath,
+        assumeSanitized,
+        strippedInternalPanel: !client.includes('id="t5"'),
+        piiGated: !assumeSanitized,
+      },
+      null,
+      2,
+    ),
+  );
+  process.exit(0);
+}
+
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  void main();
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts

@@ -0,0 +1,81 @@
+/**
+ * scripts/read-unitf-traces.ts — the STRANGLER entry point.
+ * ---------------------------------------------------------------------------
+ * Loads a handed-over UniTF `.jsonl` (one `UnifiedTrace` record per line,
+ * produced by `mutagent-cli trace fetch --export …`) → `EvalTrace[]` via the
+ * `projectUnitfToEvalTrace` adapter. This is now the ONLY trace-intake path — it
+ * replaced (and the flip DELETED) the former `ingest-sources.ts` fetcher-registry
+ * + `load-traces.ts` `mapRecord` stack: fetch/normalize now lives ENTIRELY in
+ * `mutagent-cli`; the evaluator reads a PATH and never fetches.
+ *
+ * Tolerant by design (mirrors `parseNdjsonTraces`): blank lines are skipped,
+ * malformed lines are SKIPPED + COUNTED (surfaced, never swallowed — a large
+ * export must not abort on one bad line). `parseUnitfJsonl` is PURE; only
+ * `readUnitfAsEvalTraces` touches the filesystem.
+ */
+import type { EvalTrace } from "./contracts/eval-types.ts";
+import {
+  projectUnitfToEvalTrace,
+  type UnifiedTraceLike,
+} from "./unitf-to-evaltrace.ts";
+
+export interface ParsedUnitfTraces {
+  traces: EvalTrace[];
+  /** lines that failed to parse OR lacked the minimal UniTF shape (surfaced). */
+  skipped: number;
+}
+
+/**
+ * A parsed JSON line is a usable UniTF record when it is an object carrying a
+ * non-empty `traceId` and a `spans` array — the minimal shape the projection
+ * reads. A line that parses as JSON but is not a UniTF record is counted as
+ * skipped (never silently coerced). PURE.
+ */
+function asUnitfRecord(value: unknown): UnifiedTraceLike | null {
+  if (value === null || typeof value !== "object") return null;
+  const rec = value as Record<string, unknown>;
+  if (typeof rec.traceId !== "string" || rec.traceId.length === 0) return null;
+  if (!Array.isArray(rec.spans)) return null;
+  return rec as unknown as UnifiedTraceLike;
+}
+
+/**
+ * Parse UniTF JSONL text → EvalTrace[]. One record per line; blank lines
+ * skipped; malformed / non-UniTF lines skipped + counted. PURE + deterministic.
+ */
+export function parseUnitfJsonl(text: string): ParsedUnitfTraces {
+  const traces: EvalTrace[] = [];
+  let skipped = 0;
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      skipped += 1;
+      continue;
+    }
+    const rec = asUnitfRecord(parsed);
+    if (rec === null) {
+      skipped += 1;
+      continue;
+    }
+    traces.push(projectUnitfToEvalTrace(rec));
+  }
+  return { traces, skipped };
+}
+
+/**
+ * Read a handed-over UniTF `.jsonl` file → EvalTrace[] via the projection. The
+ * effectful entry point the evaluator calls in place of `ingestSources`. THROWS
+ * only when the path is missing/unreadable — a malformed LINE inside is tolerated
+ * (skipped + counted), never fatal.
+ */
+export async function readUnitfAsEvalTraces(
+  jsonlPath: string,
+): Promise<ParsedUnitfTraces> {
+  const { readFileSync } = await import("node:fs");
+  const text = readFileSync(jsonlPath, "utf8");
+  return parseUnitfJsonl(text);
+}