@mutagent/evaluator 0.1.0-alpha.4 → 0.1.0-alpha.6

package/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts

@@ -22,6 +22,12 @@
 import type { JudgeInvoke } from "./determine-outcome.ts";
 import { determineOutcome, runJudge } from "./judge-prompt-template.ts";
 import { profileSubject, type SubjectProfile } from "./profile-subject.ts";
+import { buildSubjectProfile, deriveSubjectFrame } from "./subject-profile.ts";
+import {
+  normativeCriterionIds,
+  pendingRatifications,
+  type CalibrationDecision,
+} from "./memory/ratify.ts";
 import { splitTrainEval, buildJudgeSpec, type JudgePin } from "./build-evals.ts";
 import { rollupScorecard, type GradedCriterion, type Scorecard } from "./evaluate.ts";
 import {
@@ -64,6 +70,12 @@ export interface PipelineOptions {
   severityById?: Record<string, string>;
   /** artifact refs to enumerate on the handoff (caller-injected locators). */
   artifacts?: ArtifactRef[];
+  /**
+   * EV-4 — prior operator ratify/eliminate decisions (the review UI's `calibration.json`,
+   * routed via `memory/ratify.ts`). A NORMATIVE criterion with no `verify` here is
+   * `needs-ratification` → the verdict is HELD (surfaced in `pendingRatification`).
+   */
+  ratifications?: CalibrationDecision[];
 }
 
 export interface PipelineResult {
@@ -73,6 +85,12 @@ export interface PipelineResult {
   verdicts: CriterionVerdict[];
   scorecard: Scorecard;
   handoff: HandoverBundle;
+  /**
+   * EV-4 — normative criteria still `needs-ratification` (no operator `verify`).
+   * NON-EMPTY ⇒ the verdict is HELD: a value call the judge cannot settle is
+   * unresolved, so the run's verdict is not yet final.
+   */
+  pendingRatification: string[];
 }
 
 /** Map a determiner outcome → a sampling LabeledTrace. */
@@ -101,11 +119,20 @@ export async function runEvalPipeline(
   // EV-049 — subject profile (carries the subject vocab the engine reads).
   const profile = profileSubject(traces, opts.vocab);
 
+  // EV-5 CUT WIRE 2 — build the M1 judge-packet profile ONCE (identity · purpose ·
+  // tools + reversibility). This is `buildSubjectProfile`'s first PRODUCTION caller;
+  // it flows to the per-criterion judge preamble (EV-5). The determiner (EV-1) reads
+  // the SAME frame via `deriveSubjectFrame` so PREP (Stage A) and this pipeline
+  // (Stage B) render byte-identical determiner prompts (cross-stage cache parity).
+  const m1Profile = buildSubjectProfile({ subjectName: opts.subject.name, traces });
+  const subjectFrame = deriveSubjectFrame(opts.subject.name, traces);
+
   // EV-042 — determine an outcome per trace (sequential → deterministic order).
-  // The determiner reads the subject vocab off the profile (EV-002).
+  // The determiner reads the subject vocab off the profile (EV-002) and, EV-1, the
+  // subject frame so it derives THIS subject's expected outcome (not email signals).
   const outcomes: OutcomeResult[] = [];
   for (const t of traces) {
-    outcomes.push(await determineOutcome(t, judge, profile.vocab));
+    outcomes.push(await determineOutcome(t, judge, profile.vocab, subjectFrame));
   }
   const labeled = traces.map((t, i) => toLabeled(t, outcomes[i]));
 
@@ -125,7 +152,7 @@ export async function runEvalPipeline(
     const spec = buildJudgeSpec(criterion, exemplarsFromTrain(train), trainIds);
     // judge a representative held-out subject trace (first eval target).
     const subjectTrace = judgeTargets[0]?.trace ?? sample[0]?.trace ?? traces[0];
-    const verdict = await runJudge(spec, subjectTrace, judge, opts.pin);
+    const verdict = await runJudge(spec, subjectTrace, judge, opts.pin, m1Profile);
     verdicts.push(verdict);
     graded.push({
       criterionId: criterion.id,
@@ -171,5 +198,13 @@ export async function runEvalPipeline(
     producedAt: opts.producedAt,
   });
 
-  return { profile, outcomes, sample, verdicts, scorecard, handoff };
+  // EV-4 — a normative criterion (a value/standard call the judge cannot settle)
+  // must be operator-RATIFIED before its verdict is final. Compute which normative
+  // criteria have no `verify` decision → the verdict is HELD on them.
+  const pendingRatification = pendingRatifications(
+    normativeCriterionIds(verdicts),
+    opts.ratifications ?? [],
+  );
+
+  return { profile, outcomes, sample, verdicts, scorecard, handoff, pendingRatification };
 }

package/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts

@@ -22,11 +22,62 @@
 import {
   PROFILE_UNKNOWN,
   SubjectProfileProvenance,
+  ToolReversibility,
   type SubjectProfile,
+  type ToolReversibilityEntry,
+  type ToolReversibilityValue,
 } from "./contracts/eval-matrix.ts";
 import { inferToolInventory, inferSystemPrompt } from "./profile-subject.ts";
 import type { EvalTrace } from "./contracts/eval-types.ts";
 
+// ── EV-5 — tool reversibility classification ─────────────────────────────────
+//
+// LEAN start (operator: "classify its tools by what they do — irreversible external
+// or not; keep the start lean"). A GIVEN classification (from the subject definition)
+// ALWAYS wins; absent, a conservative NAME heuristic flags well-known irreversible-
+// external verbs and well-known read-only verbs, and marks everything else `unknown`
+// (HONEST — never a false "reversible"). The heavier per-trace side-effect harness
+// (decision option b) can supersede this later without changing the field shape.
+
+/** Verb stems whose presence in a tool name signals an IRREVERSIBLE EXTERNAL action. */
+const IRREVERSIBLE_VERBS = [
+  "send", "email", "post", "publish", "delete", "remove", "destroy", "drop",
+  "charge", "pay", "refund", "transfer", "deploy", "release", "submit", "create",
+  "insert", "write", "update", "patch", "put", "execute", "provision", "terminate",
+  "cancel", "notify", "dispatch", "message", "reply", "comment", "merge", "push",
+];
+/** Verb stems whose presence signals a READ-ONLY / safely-repeatable action. */
+const REVERSIBLE_VERBS = [
+  "get", "list", "read", "search", "fetch", "lookup", "view", "query", "find",
+  "check", "inspect", "describe", "count", "peek", "preview", "validate",
+];
+
+/**
+ * Classify one tool's reversibility (EV-5). A GIVEN `override` (subject definition)
+ * wins; else a conservative name heuristic — irreversible-external verbs first (the
+ * safety-relevant class), then read-only verbs, else `unknown` (never guessed). PURE.
+ */
+export function classifyToolReversibility(
+  name: string,
+  override?: Record<string, ToolReversibilityValue>,
+): ToolReversibilityValue {
+  const given = override?.[name];
+  if (given !== undefined) return given;
+  const n = name.toLowerCase();
+  const hits = (verbs: string[]): boolean => verbs.some((v) => n.includes(v));
+  if (hits(IRREVERSIBLE_VERBS)) return ToolReversibility.IrreversibleExternal;
+  if (hits(REVERSIBLE_VERBS)) return ToolReversibility.Reversible;
+  return ToolReversibility.Unknown;
+}
+
+/** Classify every tool in the inventory (EV-5). Deterministic; preserves order. */
+export function classifyToolList(
+  tools: string[],
+  override?: Record<string, ToolReversibilityValue>,
+): ToolReversibilityEntry[] {
+  return tools.map((name) => ({ name, reversibility: classifyToolReversibility(name, override) }));
+}
+
 /** The GIVEN facts a caller with code/metadata access can supply (M1). Any field
  *  left absent is RECONSTRUCTED from the trace batch or MARKED `unknown`. */
 export interface GivenSubjectFacts {
@@ -41,6 +92,9 @@ export interface GivenSubjectFacts {
   version?: string;
   /** the agent's system prompt (code access). Rendered COLLAPSED; never confabulated. */
   systemPrompt?: string;
+  /** EV-5 — a GIVEN per-tool reversibility classification (from the subject definition);
+   *  WINS over the name heuristic. Any tool not named here falls back to the heuristic. */
+  toolReversibility?: Record<string, ToolReversibilityValue>;
 }
 
 export interface BuildSubjectProfileParams {
@@ -117,10 +171,14 @@ export function buildSubjectProfile(params: BuildSubjectProfileParams): SubjectP
   if (systemPrompt === undefined) systemPrompt = inferSystemPrompt(traces);
   if (given?.systemPrompt === undefined) inferredFields.push("systemPrompt");
 
+  // EV-5 — classify each tool's reversibility (GIVEN override wins; else name heuristic).
+  const toolReversibility = classifyToolList(tools, given?.toolReversibility);
+
   const profile: SubjectProfile = {
     identity,
     purpose,
     tools,
+    toolReversibility,
     scope,
     harness,
     provenance: hasGiven ? SubjectProfileProvenance.Given : SubjectProfileProvenance.Reconstructed,
@@ -132,3 +190,27 @@ export function buildSubjectProfile(params: BuildSubjectProfileParams): SubjectP
   };
   return profile;
 }
+
+/** The compact determiner subject frame (EV-1) — a structural subset of the M1 profile. */
+export interface SubjectFrameShape {
+  identity: string;
+  purpose: string;
+  tools: string[];
+  toolReversibility?: ToolReversibilityEntry[];
+}
+
+/**
+ * Derive the determiner/judge subject frame (EV-1 / EV-5) from a subject name + trace
+ * batch. THE single derivation used by run-pipeline (Stage-B) AND prep (Stage-A) so
+ * the determiner prompt is byte-identical across the two stages (the cross-stage cache
+ * would miss on any divergence). PURE — same inputs ⇒ same frame.
+ */
+export function deriveSubjectFrame(subjectName: string, traces: EvalTrace[]): SubjectFrameShape {
+  const m1 = buildSubjectProfile({ subjectName, traces });
+  return {
+    identity: m1.identity,
+    purpose: m1.purpose,
+    tools: m1.tools,
+    ...(m1.toolReversibility !== undefined ? { toolReversibility: m1.toolReversibility } : {}),
+  };
+}

package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts

@@ -0,0 +1,244 @@
+/**
+ * scripts/sync-eval-criteria.ts — the EVAL-LEG reconcile CONTRACT (Wave-2 W2I5 · KP-003).
+ * ---------------------------------------------------------------------------
+ * The THIRD leg of the def → impl → eval triad. Today `#sync-spec` reconciles two
+ * legs — spec ↔ impl. When an implementation amends (the ⑤ OPTIMIZE loop's ai-engineer,
+ * or a brownfield drift), the eval criteria that GROUND the subject's evaluation can go
+ * stale w.r.t. the changed impl. This module is the EVALUATOR-SIDE CONTRACT that the
+ * reconcile (ai-architect REASONING, session-dispatched — Model-B) READS and WRITES:
+ *
+ *   1. WHICH eval leg applies — the subject-kind resolver. The eval leg's shape differs
+ *      by subject kind (this is why W2I5 was gated behind W2I1's code-quality leg):
+ *        - agent / skill / composite subject → the EVAL-SUITE criteria (the evaluator's
+ *          discovered / maintained criteria for that subject).
+ *        - code subject → the CODE-QUALITY criteria (W2I1's `DEFAULT_CODE_QUALITY_CRITERIA`
+ *          / `#mode-judge-code-quality`).
+ *   2. A deterministic STALENESS FLAG predicate — the eval-leg half of the drift trigger
+ *      (mirrors the builder freshness probe's spec/code compare; same 3-value semantics).
+ *   3. The monotonic APPLY primitive — the ai-architect reasons a criteria delta; this
+ *      applies it as criteria MAINTENANCE: upsert-by-id, append the novel, and NEVER drop
+ *      an existing criterion (EV-053, guarded by `assertMonotonicGrowth`).
+ *
+ * JUDGE-ONLY PRESERVED (EV-051): this file NEVER scores a subject and NEVER decides a
+ * subject pass/fail. It FLAGS freshness (pure epoch compare) and GROWS a criteria set
+ * (pure set algebra). The reconcile REASONING — which criteria a changed impl makes stale,
+ * what new criterion it needs — is the ai-architect's, not this code's (Model-B: code =
+ * drift predicates + deterministic maintenance only, never agent dispatch).
+ *
+ * PURE + deterministic (C-PIN): no clock / random / network; same inputs → byte-identical.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+import { assertMonotonicGrowth } from "./living-suite.ts";
+
+// ── subject kind → eval leg ──────────────────────────────────────────────────
+
+/** The subject kinds the triad reconciles. `agent|skill|composite` come from the
+ *  agentspec `definition.identity.kind`; `code` is the ⑤ OPTIMIZE code-target subject. */
+export const EvalSubjectKind = {
+  Agent: "agent",
+  Skill: "skill",
+  Composite: "composite",
+  Code: "code",
+} as const;
+export type EvalSubjectKindValue = (typeof EvalSubjectKind)[keyof typeof EvalSubjectKind];
+
+/** Which eval leg grounds a subject's evaluation — the W2I5 core rule. */
+export const EvalLegKind = {
+  /** agent / skill / composite → the evaluator's discovered/maintained eval-suite criteria. */
+  EvalSuite: "eval-suite",
+  /** code → W2I1's code-quality criteria (`DEFAULT_CODE_QUALITY_CRITERIA`). */
+  CodeQuality: "code-quality",
+} as const;
+export type EvalLegKindValue = (typeof EvalLegKind)[keyof typeof EvalLegKind];
+
+/**
+ * Resolve WHICH eval leg a subject kind reconciles against. Code subjects use the
+ * code-quality criteria (a deterministic-test-gate PLUS a quality judge); every other
+ * kind uses the discovered eval-suite criteria. This single mapping is the shape
+ * differentiator the whole triad turns on.
+ */
+export function evalLegForSubjectKind(kind: EvalSubjectKindValue): EvalLegKindValue {
+  return kind === EvalSubjectKind.Code ? EvalLegKind.CodeQuality : EvalLegKind.EvalSuite;
+}
+
+// ── the staleness flag predicate (eval-leg half of the drift trigger) ────────
+
+/** The eval-leg freshness verdict — mirrors the builder freshness probe's 3-value spec/code semantics. */
+export const EvalLegDrift = {
+  /** no eval-criteria artifact yet — the eval leg must be CONSTRUCTED (cold). */
+  MissingEval: "missing-eval",
+  /** the eval criteria are as fresh as the impl — no reconcile write needed. */
+  InSync: "in-sync",
+  /** the impl is newer than the eval criteria — the eval leg is flagged for RECONCILE. */
+  NeedsSync: "needs-sync",
+} as const;
+export type EvalLegDriftValue = (typeof EvalLegDrift)[keyof typeof EvalLegDrift];
+
+/** Effective freshness epochs (seconds) the flag predicate compares. `null` = unknown/absent. */
+export interface EvalLegFreshnessInput {
+  /** effective freshness epoch of the eval-criteria artifact; `null` = artifact absent. */
+  evalCriteriaEpoch: number | null;
+  /** effective freshness epoch of the implementation; `null` = unknown. */
+  implEpoch: number | null;
+}
+
+/**
+ * FLAG whether the eval leg is stale w.r.t. the impl. Deterministic — a pure epoch
+ * compare, identical in spirit to the builder `check-sync-spec` spec/code compare so the
+ * two legs agree on "impl is newer → reconcile". This is a FLAG only: it decides nothing
+ * about the subject and proposes no criteria — the reconcile reasoning is the agent's.
+ */
+export function flagEvalLegDrift(input: EvalLegFreshnessInput): EvalLegDriftValue {
+  if (input.evalCriteriaEpoch == null) return EvalLegDrift.MissingEval;
+  if (input.implEpoch != null && input.implEpoch > input.evalCriteriaEpoch) {
+    return EvalLegDrift.NeedsSync;
+  }
+  return EvalLegDrift.InSync;
+}
+
+/** The resolved eval leg for a subject: which criteria kind + the current staleness flag. */
+export interface EvalLegAssessment {
+  leg: EvalLegKindValue;
+  drift: EvalLegDriftValue;
+}
+
+/**
+ * Convenience: resolve the eval leg for a subject kind AND flag its staleness in one
+ * call — the exact pair the reconcile reads ("which leg, is it stale?").
+ */
+export function assessEvalLeg(
+  subjectKind: EvalSubjectKindValue,
+  freshness: EvalLegFreshnessInput,
+): EvalLegAssessment {
+  return { leg: evalLegForSubjectKind(subjectKind), drift: flagEvalLegDrift(freshness) };
+}
+
+// ── the criteria representation + the monotonic reconcile-apply ──────────────
+
+/**
+ * One eval criterion — subject-kind-agnostic (a stable `id` key + the binary `statement`,
+ * plus the `leg` it belongs to and an optional severity). Serves BOTH legs: a code-quality
+ * `Qn` criterion (`severity` ∈ critical|high|medium) and an eval-suite discovered criterion
+ * flow through the same shape. `additionalProperties: true` keeps a richer criterion (e.g.
+ * a code-quality `passDefinition`/`failDefinition`, an eval-suite `checkMethod`) forward-parsing.
+ */
+export const EvalCriterionSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    statement: Type.String({ minLength: 1 }),
+    leg: Type.Union([Type.Literal(EvalLegKind.EvalSuite), Type.Literal(EvalLegKind.CodeQuality)]),
+    severity: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: true },
+);
+export type EvalCriterion = Static<typeof EvalCriterionSchema>;
+
+/**
+ * The reconcile REQUEST the ai-architect emits: the persisted criteria `existing`, plus the
+ * agent-reasoned `proposed` additions/updates. `proposed` is an UPSERT set (append the novel,
+ * revise a same-id criterion's wording for the changed impl) — it is NEVER a delete list; the
+ * monotonic-growth invariant (EV-053) forbids dropping a criterion.
+ */
+export const EvalCriteriaReconcileRequestSchema = Type.Object(
+  {
+    subjectId: Type.String({ minLength: 1 }),
+    subjectKind: Type.Union([
+      Type.Literal(EvalSubjectKind.Agent),
+      Type.Literal(EvalSubjectKind.Skill),
+      Type.Literal(EvalSubjectKind.Composite),
+      Type.Literal(EvalSubjectKind.Code),
+    ]),
+    leg: Type.Union([Type.Literal(EvalLegKind.EvalSuite), Type.Literal(EvalLegKind.CodeQuality)]),
+    existing: Type.Array(EvalCriterionSchema),
+    proposed: Type.Array(EvalCriterionSchema),
+  },
+  { additionalProperties: false },
+);
+export type EvalCriteriaReconcileRequest = Static<typeof EvalCriteriaReconcileRequestSchema>;
+
+/** Provenance for a reconcile — pure counters (NO clock, for byte-identity / C-PIN). */
+export interface EvalCriteriaReconcileProvenance {
+  /** how many genuinely-novel criteria were appended. */
+  added: number;
+  /** how many existing criteria (same id) had their content revised. */
+  updated: number;
+  /** total criteria after the reconcile. */
+  total: number;
+}
+
+/** The reconcile RESULT — the grown/maintained criteria set + provenance. */
+export interface EvalCriteriaReconcileResult {
+  subjectId: string;
+  leg: EvalLegKindValue;
+  criteria: EvalCriterion[];
+  provenance: EvalCriteriaReconcileProvenance;
+}
+
+/** Parse + narrow a reconcile request (guarded). THROWS on schema violation. PURE. */
+export function parseEvalCriteriaReconcileRequest(value: unknown): EvalCriteriaReconcileRequest {
+  if (!Value.Check(EvalCriteriaReconcileRequestSchema, value)) {
+    const first = [...Value.Errors(EvalCriteriaReconcileRequestSchema, value)][0];
+    throw new Error(
+      `parseEvalCriteriaReconcileRequest: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid eval-criteria reconcile request"}`,
+    );
+  }
+  return value;
+}
+
+const criterionId = (c: EvalCriterion): string => c.id;
+
+/**
+ * APPLY the reconcile: maintain the eval-criteria set for a changed impl. Criteria
+ * MAINTENANCE, not judging (EV-051): upsert each `proposed` criterion by `id` (revise a
+ * same-id criterion's content in place, keeping the id), append the genuinely-novel, and
+ * assert the id-set NEVER shrank (EV-053, via the shared `assertMonotonicGrowth`). The
+ * `leg` on every proposed criterion must match the request's `leg` — mixing a code-quality
+ * criterion into an eval-suite reconcile (or vice versa) is a hard error, not a silent
+ * accept. DETERMINISTIC + PURE: existing order preserved, novel appended in proposal order;
+ * no clock / random. Same (existing, proposed) → byte-identical result.
+ */
+export function reconcileEvalCriteria(
+  request: EvalCriteriaReconcileRequest,
+): EvalCriteriaReconcileResult {
+  const req = parseEvalCriteriaReconcileRequest(request);
+
+  for (const p of req.proposed) {
+    if (p.leg !== req.leg) {
+      throw new Error(
+        `reconcileEvalCriteria: proposed criterion '${p.id}' has leg '${p.leg}' but the ` +
+          `reconcile targets leg '${req.leg}'. An eval-suite and a code-quality criterion ` +
+          "are different legs — refusing to mix them (subject-kind integrity).",
+      );
+    }
+  }
+
+  // Insertion-ordered upsert: existing order first (content possibly revised), novel appended.
+  const byId = new Map<string, EvalCriterion>();
+  for (const c of req.existing) byId.set(c.id, c);
+
+  let added = 0;
+  let updated = 0;
+  for (const p of req.proposed) {
+    if (byId.has(p.id)) {
+      byId.set(p.id, p); // revise content, id (and thus the key/order) retained
+      updated += 1;
+    } else {
+      byId.set(p.id, p); // novel → appended after the existing keys
+      added += 1;
+    }
+  }
+
+  const criteria = [...byId.values()];
+  // EV-053: a maintained criteria set may GROW or revise, but never DROP a criterion.
+  assertMonotonicGrowth(req.existing, criteria, criterionId);
+
+  return {
+    subjectId: req.subjectId,
+    leg: req.leg,
+    criteria,
+    provenance: { added, updated, total: criteria.length },
+  };
+}

package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts

@@ -164,6 +164,34 @@ function effectiveRole(span: UnitfSpan): UnitfRole {
   }
 }
 
+// ── Message-text pick (INF-1) ────────────────────────────────────────────────
+//
+// ROOT CAUSE of the Claude Code loss: the Claude Code exporter writes BOTH user
+// and assistant TEXT into `span.output` (a user turn is `kind:"event", role:"user",
+// output:<text>`; an assistant turn is `kind:"llm", role:"assistant", output:<text>`
+// with `input` UNSET). The old projection read a user/system span's `.input` — empty
+// for EVERY Claude Code trace ⇒ `input.prompt` silently empty for all 21/21.
+//
+// This VENDORS the shared reader's role-conditional pick + fallback
+// (@mutagent/tools `messagesView`, derive.ts:173-174): a user/system turn's text is
+// canonically in `input`, everything else in `output`, but the `?? input ?? output`
+// fallback recovers the text WHICHEVER field a producer used. Vendored — never
+// imported: the standalone-publish guard forbids the evaluator reaching into
+// @mutagent/tools (MIGRATION-diagnostics-evaluator.md:417); diagnostics vendors its
+// own copy and this mirrors that.
+
+/**
+ * The span's effective message TEXT: the role-conditional field pick with the
+ * `?? input ?? output` fallback (mirrors @mutagent/tools `messagesView`). Recovers
+ * a Claude Code turn (text in `output`) AND a Langfuse turn (text in `input`).
+ * PURE; "" when the span carries no text.
+ */
+function messageText(span: UnitfSpan): string {
+  const role = effectiveRole(span);
+  const primary = role === "user" || role === "system" ? span.input : span.output;
+  return asText(primary ?? span.input ?? span.output);
+}
+
 // ── Token total fallback (XF-FIX Finding D) ──────────────────────────────────
 //
 // MIRRORS diagnostics `unitf-adapter.ts` `fallbackTotalTokens` so both lanes
@@ -185,10 +213,11 @@ function fallbackTotalTokens(t: UnitfTokens | undefined): number | undefined {
  *
  *   id               ← ut.traceId
  *   name             ← ut.agentName
- *   output           ← first EFFECTIVE-assistant llm span → { response: <text of span.output> }
- *                      (XF-FIX A: role-less GENERATION spans read as assistant via kind)
- *   input            ← first user|system span's input, ELSE the response span's own
- *                      input → { prompt } (XF-FIX A: recovers the Langfuse prompt)
+ *   output           ← LAST EFFECTIVE-assistant llm span WITH text → { response: <messageText> }
+ *                      (XF-FIX A: role-less GENERATION spans read as assistant via kind;
+ *                       INF-1: last-with-text, and messageText recovers Claude Code's `output`)
+ *   input            ← first user|system span's messageText, ELSE the response span's own
+ *                      input → { prompt } (XF-FIX A: Langfuse prompt; INF-1: Claude Code `output`)
  *   observations     ← ut.spans → { type: kind.toUpperCase(), name, input, output }
  *   errored/status   ← computed.hasError ?? (status==="error" || any span error) / ut.status (XF-FIX C)
  *   tokens/totalTokens← ut.tokens split / computed.totalTokens ?? tokens.total ?? in+out (XF-FIX D)
@@ -216,31 +245,45 @@ export function projectUnitfToEvalTrace(ut: UnifiedTraceLike): EvalTrace {
 
   if (ut.agentName !== undefined) out.name = ut.agentName;
 
-  // output ← first assistant LLM span's output, wrapped to { response }. The
-  // assistant role is now EFFECTIVE (XF-FIX Finding A): a role-less GENERATION
-  // span (kind==="llm", the Langfuse shape) reads as assistant via `effectiveRole`,
-  // so its output is picked up instead of being silently dropped.
-  const responseSpan = ut.spans.find(
+  // output ← the LAST assistant LLM span WITH text, wrapped to { response }. The
+  // assistant role is EFFECTIVE (XF-FIX Finding A): a role-less GENERATION span
+  // (kind==="llm", the Langfuse shape) reads as assistant via `effectiveRole`. The
+  // second INF-1 miss: the old code took the FIRST assistant span — on Claude Code
+  // that is usually a tool_use turn with EMPTY output text, so the real reply (a
+  // LATER assistant span) was dropped. Take the LAST assistant span that carries
+  // text; the `messageText` pick recovers it from `output` (Claude Code) or `input`
+  // (whichever field the producer used).
+  const assistantSpans = ut.spans.filter(
     (s) => effectiveRole(s) === "assistant" && s.kind === "llm",
   );
-  if (responseSpan !== undefined && responseSpan.output !== undefined) {
-    out.output = { response: asText(responseSpan.output) };
+  let responseSpan: UnitfSpan | undefined;
+  for (const s of assistantSpans) {
+    if (messageText(s).length > 0) responseSpan = s; // keep the LAST with text
+  }
+  // Fall back to the last assistant span even without text, so the response SLOT is
+  // still derived from a real span (the empty case is WARNED at intake, not here).
+  const lastAssistant =
+    responseSpan ?? (assistantSpans.length > 0 ? assistantSpans[assistantSpans.length - 1] : undefined);
+  if (responseSpan !== undefined) {
+    out.output = { response: messageText(responseSpan) };
   }
 
-  // input ← first user|system span's input, wrapped to { prompt }. A role-less
-  // span never infers to user/system, so when there is no explicit user/system
-  // turn (again the Langfuse shape — one GENERATION span carrying prompt in its
-  // OWN `input` and response in `output`) the prompt is recovered from the
-  // response span's `input`. Without this fallback `input.prompt` was empty for
-  // EVERY Langfuse trace (Finding A).
+  // input ← the first user|system span's TEXT, wrapped to { prompt }. `messageText`
+  // recovers the text from `output` (Claude Code writes the user turn there) or
+  // `input` (Langfuse). When there is no explicit user/system turn (the Langfuse
+  // shape — one GENERATION span carrying the prompt in its OWN `input` and the
+  // response in `output`) the prompt is recovered from the response span's `input`
+  // SPECIFICALLY (not `messageText`, which for an assistant span would return the
+  // response). Without these fallbacks `input.prompt` was empty for every Claude
+  // Code trace (INF-1) and every Langfuse trace (Finding A).
   const promptSpan = ut.spans.find(
     (s) => effectiveRole(s) === "user" || effectiveRole(s) === "system",
   );
-  if (promptSpan !== undefined && promptSpan.input !== undefined) {
-    out.input = { prompt: asText(promptSpan.input) };
-  } else if (responseSpan !== undefined && responseSpan.input !== undefined) {
-    out.input = { prompt: asText(responseSpan.input) };
+  let promptText = promptSpan !== undefined ? messageText(promptSpan) : "";
+  if (promptText.length === 0 && lastAssistant !== undefined && lastAssistant.input !== undefined) {
+    promptText = asText(lastAssistant.input);
   }
+  if (promptText.length > 0) out.input = { prompt: promptText };
 
   // Finding C — carry structured error/status so an error-KIND criterion has a
   // field to read (previously error state survived only as textual span output).