@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/substrate.ts

@@ -0,0 +1,162 @@
+/**
+ * scripts/substrate.ts — EV-050 judge-runtime selection.
+ * ---------------------------------------------------------------------------
+ * NOTE (v0.2.0 rename): `substrate` → `judge_runtime` in config + the public API
+ * (`resolveSubstrate`→`resolveJudgeRuntime`, `judgeForSubstrate`→`judgeForRuntime`).
+ * The enum VALUES are UNCHANGED (`agent-dispatch`/`in-house`/`code-based`/
+ * `user-framework`) and the EV-050 `Substrate` enum symbol (contracts/eval-types.ts)
+ * is kept — only the config KEY + the two selector functions were renamed.
+ *
+ * The W1 DEFAULT judge-runtime is **agent-dispatch** (scripts/agent-dispatch.ts):
+ * verdicts are produced by parent-session-dispatched eval-judge / error-analyst
+ * leaf subagents reasoning on the HOST runtime (Claude Code, diagnostics-style
+ * mass-dispatch) and read back from verdict FILES — the default path calls NO
+ * provider SDK. The onboarding choice (PRD §7.9) stays a real fork; this picks
+ * the DEFAULT the spine + dogfood build against, and STRUCTURES the rest as
+ * seams rather than hardcoding any single transport:
+ *
+ *   - agent-dispatch (DEFAULT) → a verdict-file-backed JudgeInvoke; the LLM
+ *                      reasoning happens in dispatched subagents on the host,
+ *                      NOT in a script. Requires `verdictDir` (where the
+ *                      dispatched subagents wrote their verdict files).
+ *   - in-house       (OPTIONAL) → a live LLM JudgeInvoke (google-genai shape);
+ *                      kept, not deleted — DEMOTED from default. For the CI /
+ *                      code-based export path that wants a provider call.
+ *   - code-based     → objective checks over tool outputs; runs WITHOUT an LLM
+ *                      judge → judgeForRuntime THROWS (deterministic code).
+ *   - user-framework → an EXPORT target (Vitest/promptfoo/Braintrust/…); the
+ *                      evaluator emits to it at onboarding → not a live judge.
+ *
+ * Why agent-dispatch is the default (operator correction 2026-06-19): the
+ * Evaluator's functions must be performed BY an Evaluator agent on Claude Code
+ * itself, mass-dispatched for throughput (mirror diagnostics), NOT by a script
+ * calling Gemini. In-house Gemini remains a real onboarding option (EV-050) but
+ * is no longer the spine default. Model intent stays sacred either way — the
+ * host's pinned model for agent-dispatch, the asserted provider model in-house.
+ */
+import { Substrate, type SubstrateValue } from "./contracts/eval-types.ts";
+import { createInHouseJudge } from "./judge-provider.ts";
+import { createAgentDispatchJudge } from "./agent-dispatch.ts";
+import type { JudgeInvoke } from "./determine-outcome.ts";
+
+const KNOWN_SUBSTRATES: ReadonlySet<string> = new Set([
+  Substrate.AgentDispatch,
+  Substrate.InHouse,
+  Substrate.UserFramework,
+  Substrate.CodeBased,
+]);
+
+/** Resolve the judge-runtime choice; default = agent-dispatch. Unknown → THROW (no silent default). */
+export function resolveJudgeRuntime(choice?: string): SubstrateValue {
+  if (choice === undefined || choice === "") return Substrate.AgentDispatch;
+  if (!KNOWN_SUBSTRATES.has(choice)) {
+    throw new Error(
+      `resolveJudgeRuntime: unknown judge_runtime '${choice}'. Choose one of ` +
+        `{${[...KNOWN_SUBSTRATES].join(", ")}}.`,
+    );
+  }
+  return choice as SubstrateValue;
+}
+
+export interface SubstrateDescription {
+  kind: SubstrateValue;
+  /** true for transports that yield a live JudgeInvoke (agent-dispatch · in-house);
+   *  false for the seams (code-based · user-framework). */
+  liveJudge: boolean;
+  /** the verdict transport: where the judge text comes from. */
+  transport: "agent-dispatch" | "provider-sdk" | "code" | "export";
+  /** true ONLY for the in-house provider-SDK path (the others never call a provider). */
+  callsProvider: boolean;
+  note: string;
+}
+
+/** Describe a substrate (which one provides a live judge, and via what transport). */
+export function describeSubstrate(s: SubstrateValue): SubstrateDescription {
+  switch (s) {
+    case Substrate.AgentDispatch:
+      return {
+        kind: s,
+        liveJudge: true,
+        transport: "agent-dispatch",
+        callsProvider: false,
+        note: "Claude Code agent-dispatch (DEFAULT) — verdicts from host-runtime subagents (verdict files), no provider",
+      };
+    case Substrate.InHouse:
+      return {
+        kind: s,
+        liveJudge: true,
+        transport: "provider-sdk",
+        callsProvider: true,
+        note: "in-house AI-SDK judge (OPTIONAL — demoted from default; provider-SDK call)",
+      };
+    case Substrate.CodeBased:
+      return {
+        kind: s,
+        liveJudge: false,
+        transport: "code",
+        callsProvider: false,
+        note: "objective code-checks over tool outputs — no LLM judge",
+      };
+    case Substrate.UserFramework:
+    default:
+      return {
+        kind: s,
+        liveJudge: false,
+        transport: "export",
+        callsProvider: false,
+        note: "export target (user's eval framework) — emit, not a live judge",
+      };
+  }
+}
+
+export interface SubstrateJudgeConfig {
+  substrate: SubstrateValue;
+  /** agent-dispatch: the dir the dispatched subagents wrote their verdict files into. */
+  verdictDir?: string;
+  /** in-house: the resolved (pinned) provider model. */
+  model?: string;
+  apiKey?: string;
+}
+
+/**
+ * Return a live `JudgeInvoke` for the judge-runtime.
+ *   - agent-dispatch (DEFAULT) → a verdict-file-backed judge (requires verdictDir);
+ *     NO provider is loaded.
+ *   - in-house (OPTIONAL) → the google-genai judge (requires a resolved model —
+ *     model intent sacred; createInHouseJudge asserts it).
+ *   - code-based / user-framework → SEAMS; THROW (no live judge to return).
+ */
+export function judgeForRuntime(cfg: SubstrateJudgeConfig): JudgeInvoke {
+  if (cfg.substrate === Substrate.AgentDispatch) {
+    if (cfg.verdictDir === undefined || cfg.verdictDir === "") {
+      throw new Error(
+        "judgeForRuntime(agent-dispatch): a verdictDir is required — it is the " +
+          "directory the parent session's dispatched eval-judge / error-analyst " +
+          "subagents wrote their verdict files into. PREP → dispatch → collect, " +
+          "then aggregate (references/workflows/orchestrator-protocol.md).",
+      );
+    }
+    return createAgentDispatchJudge({ verdictDir: cfg.verdictDir });
+  }
+  if (cfg.substrate === Substrate.InHouse) {
+    if (cfg.model === undefined || cfg.model === "") {
+      throw new Error(
+        "judgeForRuntime(in-house): a model is required (model intent sacred). " +
+          "Resolve it via resolveJudgeModel() and pass it through.",
+      );
+    }
+    return createInHouseJudge({ model: cfg.model, ...(cfg.apiKey !== undefined ? { apiKey: cfg.apiKey } : {}) });
+  }
+  if (cfg.substrate === Substrate.CodeBased) {
+    throw new Error(
+      "judgeForRuntime(code-based): code-based evals run WITHOUT an LLM judge — " +
+        "there is no live judge to return. Run the deterministic code-checks " +
+        "instead (this is a seam, not a live-judge path).",
+    );
+  }
+  throw new Error(
+    "judgeForRuntime(user-framework): the user's framework is an EXPORT seam, " +
+      "not a live judge. Emit the suite to the framework at onboarding; do not " +
+      "expect a JudgeInvoke here.",
+  );
+}

package/.claude/skills/mutagent-evaluator/scripts/ui-slots.ts

@@ -0,0 +1,119 @@
+/**
+ * scripts/ui-slots.ts — EV-039/040 HTML-artifact missing-data audit (Code-only).
+ * ---------------------------------------------------------------------------
+ * The operator's first-class case: *"if a skill/agent produces an HTML artifact,
+ * data-leak is concerned with MISSING DATA in the UI representation."* A value
+ * the agent COMPUTED but did NOT render is a missing-data leak.
+ *
+ * This is the deterministic, SUBJECT-AGNOSTIC half of the EV-039/040 Hybrid: it
+ * cross-references a PROFILE-SUPPLIED expected-UI-slot list (EV-037
+ * `expectedUiSlots`, NOT the v1 diagnostics-specific hardcoded slot names) against
+ * the agent's computed values + the published HTML, and classifies each slot:
+ *   - computed-but-not-rendered (EV-039) — a value computed yet ABSENT from the
+ *     HTML → missing-data leak (locus UI, cls B). The operator's exact case;
+ *     works on the HTML-only path (the v1 OUTSIDER FALLBACK promoted to
+ *     first-class — no intermediate render-input file required).
+ *   - orphan slot (EV-039) — the HTML references the slot but NO producer
+ *     computed it → drawn-but-empty.
+ *   - faithful — computed AND rendered verbatim → explicitly NOT a leak.
+ * Nuanced faithfulness (altered/truncated, EV-040) is left to the judge — this
+ * helper flags only verbatim presence/absence (it decides no severity).
+ *
+ * PURE + deterministic: no clock / random / network; same inputs → same audit.
+ */
+
+/** Classification of one expected UI slot. */
+export const UiSlotKind = {
+  /** Computed by the agent but ABSENT from the rendered HTML (EV-039 leak). */
+  ComputedNotRendered: "computed-not-rendered",
+  /** Referenced in the HTML but NO producer computed it (EV-039 orphan). */
+  OrphanSlot: "orphan-slot",
+  /** Computed AND rendered verbatim — NOT a leak. */
+  Faithful: "faithful",
+  /** Neither computed nor referenced in the HTML — out of scope this run. */
+  Absent: "absent",
+} as const;
+export type UiSlotKindValue = (typeof UiSlotKind)[keyof typeof UiSlotKind];
+
+export interface UiSlotFinding {
+  slot: string;
+  kind: UiSlotKindValue;
+  /** the computed value string (when the agent produced one). */
+  computedValue?: string;
+  /** whether the slot is referenced (by name) anywhere in the HTML. */
+  slotReferenced: boolean;
+  /** whether the computed value appears verbatim in the HTML. */
+  valueRendered: boolean;
+}
+
+export interface UiSlotAuditInput {
+  /** EV-037 profile-supplied expected UI slots (subject-agnostic). */
+  expectedSlots: string[];
+  /** slot -> the value the agent COMPUTED (from trace / runMeta / intermediate). */
+  computedSlots: Record<string, string>;
+  /** the published HTML artifact text (the HTML-only path is supported). */
+  renderedHtml: string;
+}
+
+export interface UiSlotAudit {
+  findings: UiSlotFinding[];
+  /** EV-039 leaks: computed but not in the HTML. */
+  computedNotRendered: string[];
+  /** EV-039 orphans: referenced in the HTML but never computed. */
+  orphanSlots: string[];
+}
+
+/** Does the HTML reference this slot by name (a token match)? */
+function slotReferenced(html: string, slot: string): boolean {
+  if (slot.length === 0) return false;
+  return html.includes(slot);
+}
+
+/** Does the computed value appear verbatim in the HTML? */
+function valueRendered(html: string, value: string): boolean {
+  const v = value.trim();
+  if (v.length === 0) return false;
+  return html.includes(v);
+}
+
+/**
+ * Audit the expected UI slots against the computed values + the rendered HTML.
+ * PURE + deterministic. SUBJECT-AGNOSTIC — the slot list is supplied, never a
+ * module constant. Findings are emitted in `expectedSlots` order (stable).
+ */
+export function auditUiSlots(input: UiSlotAuditInput): UiSlotAudit {
+  const findings: UiSlotFinding[] = [];
+  const computedNotRendered: string[] = [];
+  const orphanSlots: string[] = [];
+
+  for (const slot of input.expectedSlots) {
+    const hasComputed = Object.prototype.hasOwnProperty.call(input.computedSlots, slot);
+    const computedValue = hasComputed ? input.computedSlots[slot] : undefined;
+    const referenced = slotReferenced(input.renderedHtml, slot);
+    const rendered =
+      computedValue !== undefined && valueRendered(input.renderedHtml, computedValue);
+
+    let kind: UiSlotKindValue;
+    if (hasComputed && !rendered) {
+      kind = UiSlotKind.ComputedNotRendered; // EV-039: computed but missing from HTML
+      computedNotRendered.push(slot);
+    } else if (!hasComputed && referenced) {
+      kind = UiSlotKind.OrphanSlot; // EV-039: drawn but no producer
+      orphanSlots.push(slot);
+    } else if (hasComputed && rendered) {
+      kind = UiSlotKind.Faithful;
+    } else {
+      kind = UiSlotKind.Absent;
+    }
+
+    findings.push({
+      slot,
+      kind,
+      ...(computedValue !== undefined ? { computedValue } : {}),
+      slotReferenced: referenced,
+      valueRendered: rendered,
+    });
+  }
+
+  return { findings, computedNotRendered, orphanSlots };
+}

package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts

@@ -0,0 +1,284 @@
+/**
+ * scripts/unitf-to-evaltrace.ts — the STRANGLER projection adapter.
+ * ---------------------------------------------------------------------------
+ * Projects one UniTF `UnifiedTrace` record → the evaluator's in-package
+ * `EvalTrace` shape (MIGRATION-diagnostics-evaluator.md §3.2). This REPLACED (and
+ * the flip DELETED) `load-traces.ts` `mapRecord` (raw Langfuse → EvalTrace):
+ * instead of mapping a raw platform record, the evaluator reads a PRE-NORMALIZED
+ * UniTF JSONL handed over by `mutagent-cli` and projects each record here. All
+ * downstream (sample-traces / profile-subject / discover / judge / scorecard)
+ * consumes the resulting `EvalTrace[]` UNCHANGED.
+ *
+ * STANDALONE — the UniTF type shape is PORTED (a minimal structural subset), NOT
+ * imported from `@mutagent/tools`. The boundary between the CLI and this skill is
+ * the JSONL file on disk, never a source import (avoids a cross-package cycle;
+ * see the migration doc §6.1 "Circular import" risk row). Only the fields the
+ * projection actually reads are ported.
+ *
+ * PURE + deterministic: no clock, no random, no I/O. (The effectful JSONL read
+ * lives in `read-unitf-traces.ts`, which calls this.)
+ */
+import type { EvalTrace, TraceObservation } from "./contracts/eval-types.ts";
+
+// ── Ported minimal UniTF shape (structural subset of @mutagent/tools unitf.ts) ──
+//
+// Intentionally a LOOSE structural mirror — a `UnifiedTrace` handed over on disk
+// carries the full frozen schema, but the projection reads only these fields.
+// Kept in-package so the skill never source-imports the tools package.
+
+/** The message-style role a UniTF span may carry (transcript projection). */
+export type UnitfRole = "user" | "assistant" | "system" | "tool" | "toolResult";
+
+/** The OTel-aligned span kind. Kept as the literal union the projection branches on. */
+export type UnitfSpanKind =
+  | "llm"
+  | "tool"
+  | "retrieval"
+  | "agent"
+  | "event"
+  | "span";
+
+/** The OTel-aligned status a UniTF span (and the trace) may carry. */
+export type UnitfStatus = "ok" | "error" | "unknown";
+
+/** Token usage split (structural subset of @mutagent/tools `TokensSchema`). */
+export interface UnitfTokens {
+  input?: number;
+  output?: number;
+  total?: number;
+}
+
+/** One UniTF span (unifies a message AND an observation). Structural subset. */
+export interface UnitfSpan {
+  name: string;
+  kind: UnitfSpanKind;
+  role?: UnitfRole;
+  /** XF-FIX Finding C — span-level status, used to derive the trace `errored` flag. */
+  status?: UnitfStatus;
+  input?: unknown;
+  output?: unknown;
+}
+
+/** A raw platform score, carried through verbatim (Langfuse-style). */
+export interface UnitfScore {
+  name: string;
+  value?: number | string | boolean;
+  comment?: string;
+  dataType?: string;
+}
+
+/** The `ext.eval` fidelity block the fidelity gate reads. Structural subset. */
+export interface UnitfEvalExt {
+  incomplete?: boolean;
+  incompleteReason?: string;
+}
+
+/** The derived `computed` convenience block (structural subset). NEVER authoritative —
+ *  read only as a fallback, exactly as the diagnostics adapter does. */
+export interface UnitfComputed {
+  hasError?: boolean;
+  totalTokens?: number;
+}
+
+/** One UniTF record. Structural subset of the frozen `UnifiedTrace` contract. */
+export interface UnifiedTraceLike {
+  traceId: string;
+  agentName?: string;
+  latencyMs?: number;
+  costUsd?: number;
+  tags?: string[];
+  scores?: UnitfScore[];
+  spans: UnitfSpan[];
+  /** XF-FIX Finding C — top-level trace status (`ok|error|unknown`). */
+  status?: UnitfStatus;
+  /** XF-FIX Finding D — trace-level token usage. */
+  tokens?: UnitfTokens;
+  /** XF-FIX Finding D — derived convenience block (hasError / totalTokens fallbacks). */
+  computed?: UnitfComputed;
+  ext?: {
+    eval?: UnitfEvalExt;
+  };
+}
+
+// ── Text derivation (ported from @mutagent/tools derive.ts `asText`) ─────────
+//
+// A span's input/output is `unknown` — real exports carry nested payloads. The
+// evaluator's `EvalTrace.input.prompt` / `output.response` are strings, so a
+// deterministic best-effort stringify is needed. Mirrors the tools-package
+// `asText` (pull a known text-ish field, else stable JSON) so a value projected
+// here matches what a diagnostics `messagesView` would render.
+
+/** Best-effort deterministic stringify of a span payload. PURE. */
+function asText(v: unknown): string {
+  if (v === undefined || v === null) return "";
+  if (typeof v === "string") return v;
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  if (typeof v === "object") {
+    const o = v as Record<string, unknown>;
+    for (const k of ["text", "content", "response", "prompt", "message"]) {
+      if (typeof o[k] === "string") return o[k] as string;
+    }
+    try {
+      return JSON.stringify(v);
+    } catch {
+      return "";
+    }
+  }
+  return "";
+}
+
+// ── Role kind-fallback (XF-FIX Finding A) ────────────────────────────────────
+//
+// ROOT CAUSE of the Langfuse loss: `langfuse.ts obsToSpan` never sets `span.role`
+// (a Langfuse observation carries no message role). The old projection derived
+// the prompt from `role∈{user,system}` and the response from `role==="assistant"
+// && kind==="llm"` — BOTH miss on a role-less span ⇒ `input.prompt` / `output.
+// response` were empty for EVERY Langfuse trace. Diagnostics never had this bug:
+// its `unitf-adapter.ts` `spanMessageRole` falls back to `kind` when `role` is
+// absent. This mirrors that fallback so the evaluator lane survives the identical
+// role-less input Diagnostics already reads fine (reader-side fix — the shared
+// `langfuse.ts` producer is deliberately NOT touched, keeping the clean
+// Diagnostics lane untouched).
+
+/**
+ * The span's effective message role. When an explicit `role` is present it wins;
+ * otherwise it is inferred from `kind` (a role-less `llm`/`agent` span — the
+ * Langfuse GENERATION shape — reads as `assistant`; a `tool` span as `tool`).
+ * MIRRORS diagnostics `unitf-adapter.ts` `spanMessageRole`. PURE.
+ *
+ * NOTE: inference never yields `user`/`system` (a role-less span is never a user
+ * turn), so prompt detection still requires an EXPLICIT user/system role — the
+ * Langfuse prompt is recovered from the GENERATION span's own `input` instead
+ * (see the response-span input fallback in the projection).
+ */
+function effectiveRole(span: UnitfSpan): UnitfRole {
+  if (span.role !== undefined) return span.role;
+  switch (span.kind) {
+    case "llm":
+    case "agent":
+      return "assistant";
+    case "tool":
+      return "tool";
+    default:
+      return "assistant";
+  }
+}
+
+// ── Token total fallback (XF-FIX Finding D) ──────────────────────────────────
+//
+// MIRRORS diagnostics `unitf-adapter.ts` `fallbackTotalTokens` so both lanes
+// report the SAME total: prefer `tokens.total`, else `input + output` when either
+// is present, else undefined (NEVER 0 — an absent count stays absent).
+
+/** Best-effort trace total from a `UnitfTokens` split. PURE; undefined-safe. */
+function fallbackTotalTokens(t: UnitfTokens | undefined): number | undefined {
+  if (t === undefined) return undefined;
+  if (t.total !== undefined) return t.total;
+  if (t.input !== undefined || t.output !== undefined) {
+    return (t.input ?? 0) + (t.output ?? 0);
+  }
+  return undefined;
+}
+
+/**
+ * Project a UniTF record → EvalTrace (migration doc §3.2 mapping table).
+ *
+ *   id               ← ut.traceId
+ *   name             ← ut.agentName
+ *   output           ← first EFFECTIVE-assistant llm span → { response: <text of span.output> }
+ *                      (XF-FIX A: role-less GENERATION spans read as assistant via kind)
+ *   input            ← first user|system span's input, ELSE the response span's own
+ *                      input → { prompt } (XF-FIX A: recovers the Langfuse prompt)
+ *   observations     ← ut.spans → { type: kind.toUpperCase(), name, input, output }
+ *   errored/status   ← computed.hasError ?? (status==="error" || any span error) / ut.status (XF-FIX C)
+ *   tokens/totalTokens← ut.tokens split / computed.totalTokens ?? tokens.total ?? in+out (XF-FIX D)
+ *   scores           ← ut.scores (verbatim)
+ *   tags             ← ut.tags
+ *   latencyMs/costUsd← ut.latencyMs / ut.costUsd
+ *   incomplete       ← ut.ext.eval.incomplete ?? false (the §9.4.2 fidelity marker)
+ *   incompleteReason ← ut.ext.eval.incompleteReason
+ *
+ * PURE + deterministic — same record ⇒ same EvalTrace.
+ */
+export function projectUnitfToEvalTrace(ut: UnifiedTraceLike): EvalTrace {
+  const observations: TraceObservation[] = ut.spans.map((span) => {
+    const obs: TraceObservation = { type: span.kind.toUpperCase() };
+    if (span.name !== undefined) obs.name = span.name;
+    if (span.input !== undefined) obs.input = span.input;
+    if (span.output !== undefined) obs.output = span.output;
+    return obs;
+  });
+
+  const out: EvalTrace = {
+    id: ut.traceId,
+    observations,
+  };
+
+  if (ut.agentName !== undefined) out.name = ut.agentName;
+
+  // output ← first assistant LLM span's output, wrapped to { response }. The
+  // assistant role is now EFFECTIVE (XF-FIX Finding A): a role-less GENERATION
+  // span (kind==="llm", the Langfuse shape) reads as assistant via `effectiveRole`,
+  // so its output is picked up instead of being silently dropped.
+  const responseSpan = ut.spans.find(
+    (s) => effectiveRole(s) === "assistant" && s.kind === "llm",
+  );
+  if (responseSpan !== undefined && responseSpan.output !== undefined) {
+    out.output = { response: asText(responseSpan.output) };
+  }
+
+  // input ← first user|system span's input, wrapped to { prompt }. A role-less
+  // span never infers to user/system, so when there is no explicit user/system
+  // turn (again the Langfuse shape — one GENERATION span carrying prompt in its
+  // OWN `input` and response in `output`) the prompt is recovered from the
+  // response span's `input`. Without this fallback `input.prompt` was empty for
+  // EVERY Langfuse trace (Finding A).
+  const promptSpan = ut.spans.find(
+    (s) => effectiveRole(s) === "user" || effectiveRole(s) === "system",
+  );
+  if (promptSpan !== undefined && promptSpan.input !== undefined) {
+    out.input = { prompt: asText(promptSpan.input) };
+  } else if (responseSpan !== undefined && responseSpan.input !== undefined) {
+    out.input = { prompt: asText(responseSpan.input) };
+  }
+
+  // Finding C — carry structured error/status so an error-KIND criterion has a
+  // field to read (previously error state survived only as textual span output).
+  // `errored` mirrors diagnostics `computed.hasError ?? status==="error"`, extended
+  // to also honor any span-level error. Always set (false = grounded not-errored).
+  if (ut.status !== undefined) out.status = ut.status;
+  out.errored =
+    ut.computed?.hasError ??
+    (ut.status === "error" || ut.spans.some((s) => s.status === "error"));
+
+  // Finding D — carry token usage (undefined-safe; never coerced to 0) so a
+  // token-budget criterion is supported at intake, alongside the already-carried
+  // costUsd + latencyMs. `totalTokens` uses the diagnostics fallback chain.
+  if (ut.tokens !== undefined) {
+    const tk: { input?: number; output?: number; total?: number } = {};
+    if (typeof ut.tokens.input === "number") tk.input = ut.tokens.input;
+    if (typeof ut.tokens.output === "number") tk.output = ut.tokens.output;
+    if (typeof ut.tokens.total === "number") tk.total = ut.tokens.total;
+    if (tk.input !== undefined || tk.output !== undefined || tk.total !== undefined) {
+      out.tokens = tk;
+    }
+  }
+  const totalTokens = ut.computed?.totalTokens ?? fallbackTotalTokens(ut.tokens);
+  if (totalTokens !== undefined) out.totalTokens = totalTokens;
+
+  if (ut.scores !== undefined) out.scores = ut.scores;
+  if (ut.tags !== undefined) {
+    out.tags = ut.tags.filter((t): t is string => typeof t === "string");
+  }
+  if (typeof ut.latencyMs === "number") out.latencyMs = ut.latencyMs;
+  if (typeof ut.costUsd === "number") out.costUsd = ut.costUsd;
+
+  // §9.4.2 fidelity marker — projected DIRECTLY from ext.eval so an incomplete
+  // trace never reaches the criteria walk unflagged.
+  const incomplete = ut.ext?.eval?.incomplete ?? false;
+  if (incomplete) out.incomplete = true;
+  const reason = ut.ext?.eval?.incompleteReason;
+  if (typeof reason === "string" && reason.length > 0) out.incompleteReason = reason;
+
+  return out;
+}