@iowarp/clio-coder 0.4.1 → 0.4.3

package/src/domains/eval/artifacts/store.ts

@@ -11,6 +11,11 @@ import { parseEvalVerdictEnvelopeV1 } from "../schema/verdict.js";
 import { evalRoot } from "../store.js";
 import { redactArtifactForStorage } from "./redact.js";
 
+// This is the documented location (docs/architecture/artifact-versions.md's
+// Eval Artifact row): <dataDir>/evals/<evalId>.json, flat. The legacy version-1
+// shape skills-eval writes lives at evalRoot()/skills-eval/ instead
+// (store.ts's evalArtifactPath) so the two writers can never resolve to the
+// same file for the same id.
 function evalArtifactPathV4(dataDir: string, evalId: string): string {
 	assertSafeId(evalId, "eval");
 	return join(evalRoot(dataDir), `${evalId}.json`);

package/src/domains/eval/metrics/call-ledger-stream.ts

@@ -6,7 +6,11 @@ export interface EvalCallLedgerFold {
 	entries(): SessionEntry[];
 }
 
-/** Fold structured assistant completions into the eval's per-call ledger. */
+/**
+ * Fold stdout assistant completions into a fallback ledger. These spans start
+ * at the provider's message_start, which may follow its response-header wait;
+ * they are not complete request latency. Native eval prefers session timing.
+ */
 export function createEvalCallLedgerFold(now: () => number = () => performance.now()): EvalCallLedgerFold {
 	const entries: SessionEntry[] = [];
 	let pending = "";
@@ -21,7 +25,7 @@ export function createEvalCallLedgerFold(now: () => number = () => performance.n
 			activeFirstOutputAt = null;
 			return;
 		}
-		if (event.type === "message_update" && activeStartedAt !== null && activeFirstOutputAt === null) {
+		if (isAssistantOutput(event) && activeStartedAt !== null && activeFirstOutputAt === null) {
 			activeFirstOutputAt = now();
 			return;
 		}
@@ -42,19 +46,25 @@ export function createEvalCallLedgerFold(now: () => number = () => performance.n
 		};
 		if (isRecord(message.backendTimings)) promptCache.backend = structuredClone(message.backendTimings);
 		const previous = entries.at(-1);
+		const timestamp = messageTimestamp(message.timestamp);
 		entries.push({
 			kind: "message",
 			role: "assistant",
 			turnId: `eval-call-${entries.length + 1}`,
 			parentTurnId: previous?.turnId ?? null,
-			timestamp: messageTimestamp(message.timestamp),
+			timestamp: timestamp ?? new Date(0).toISOString(),
 			payload: {
+				// Retain the legacy ISO placeholder without treating it as observed
+				// chronology. Per-call monotonic timing remains usable on its own.
+				...(timestamp === null ? { timestampEstimated: true } : {}),
 				promptCache,
-				timing: {
-					ttftMs:
-						activeStartedAt === null ? null : Math.round(Math.max(0, (activeFirstOutputAt ?? endedAt) - activeStartedAt)),
-					apiMs: activeStartedAt === null ? 0 : Math.round(Math.max(0, endedAt - activeStartedAt)),
-				},
+				timing:
+					activeStartedAt === null
+						? null
+						: {
+								ttftMs: activeFirstOutputAt === null ? null : Math.round(Math.max(0, activeFirstOutputAt - activeStartedAt)),
+								apiMs: Math.round(Math.max(0, endedAt - activeStartedAt)),
+							},
 				usage: structuredClone(usage),
 			},
 		});
@@ -82,15 +92,28 @@ export function createEvalCallLedgerFold(now: () => number = () => performance.n
 	};
 }
 
+function isAssistantOutput(event: Record<string, unknown>): boolean {
+	// Headless chat projects updates to text/thinking deltas; worker stdout
+	// retains the nested update. Structural events are not token observations.
+	const output = event.type === "message_update" ? event.assistantMessageEvent : event;
+	if (!isRecord(output)) return false;
+	if (output.type === "toolcall_start") return event.type === "message_update";
+	return (
+		(output.type === "text_delta" || output.type === "thinking_delta" || output.type === "toolcall_delta") &&
+		typeof output.delta === "string" &&
+		output.delta.length > 0
+	);
+}
+
 function isAssistantMessage(value: unknown): value is Record<string, unknown> & { role: "assistant" } {
 	return isRecord(value) && value.role === "assistant";
 }
 
-function messageTimestamp(value: unknown): string {
+function messageTimestamp(value: unknown): string | null {
 	const milliseconds = nonNegativeNumber(value);
-	if (milliseconds === null) return new Date(0).toISOString();
+	if (milliseconds === null) return null;
 	const date = new Date(milliseconds);
-	return Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString();
+	return Number.isFinite(date.getTime()) ? date.toISOString() : null;
 }
 
 function nonNegativeNumber(value: unknown): number | null {

package/src/domains/eval/metrics/token-stream.ts

@@ -1,14 +1,17 @@
+import { extractReasoningTokens } from "../../session/context-accounting.js";
 import type { EvalTokenMetricsV4 } from "../schema/artifact.js";
 
 export interface EvalTokenStreamUsage {
 	/**
 	 * True when the stream carried at least one completed assistant message
-	 * with provider usage. False means this runner observed no token
+	 * with observed token facts. False means this runner observed no token
 	 * accounting at all, which is not the same as a run that cost nothing.
 	 */
 	measured: boolean;
+	/** Known token subtotals; missing failed-call coverage is reported separately. */
 	tokens: EvalTokenMetricsV4;
 	costUsd: number;
+	provider: EvalProviderObservation;
 }
 
 export interface EvalTokenUsageFold {
@@ -21,6 +24,7 @@ export const UNMEASURED_TOKEN_USAGE: EvalTokenStreamUsage = {
 	measured: false,
 	tokens: { input: 0, output: 0, total: 0, cacheRead: 0, cacheWrite: 0 },
 	costUsd: 0,
+	provider: emptyProviderObservation(),
 };
 
 /**
@@ -30,8 +34,12 @@ export const UNMEASURED_TOKEN_USAGE: EvalTokenStreamUsage = {
  * carrying a completed message's usage exactly once; `turn_end` republishes
  * the same assistant message and `agent_end` republishes its segment's
  * summary, so counting those too would multiply a run's reported cost. Every
- * message counts, because a headless turn spans several agent segments and the
- * run's cost is their sum.
+ * reported call counts, including known errored-call spend: a headless turn
+ * spans several segments and its cost is their sum. Provider facts and the
+ * errored known share are folded here too, from the same events before truncation.
+ * Failed all-zero usage is ambiguous (adapters synthesize it), so it cannot
+ * establish measured usage. Unknown failed spend is retained as missing coverage,
+ * never subtracted from inclusive totals or used to change task pass policy.
  *
  * Folding as the stream arrives is what makes the count trustworthy: the
  * operator-facing stdout artifact keeps only a bounded head and tail, so a
@@ -42,6 +50,7 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 	let costUsd = 0;
 	let measured = false;
 	let pending = "";
+	const provider = emptyProviderObservation();
 
 	const consume = (line: string): void => {
 		if (line.trim().length === 0) return;
@@ -51,23 +60,16 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 		} catch {
 			return;
 		}
-		if (!isRecord(event) || event.type !== "message_end") return;
-		const message = isRecord(event.message) ? event.message : undefined;
-		if (message === undefined || message.role !== "assistant") return;
-		const usage = isRecord(message.usage) ? message.usage : undefined;
-		if (usage === undefined) return;
-		measured = true;
-		const input = numberField(usage, "input");
-		const output = numberField(usage, "output");
-		const cacheRead = numberField(usage, "cacheRead");
-		const cacheWrite = numberField(usage, "cacheWrite");
-		const totalTokens = numberField(usage, "totalTokens");
-		tokens.input += input;
-		tokens.output += output;
-		tokens.cacheRead += cacheRead;
-		tokens.cacheWrite += cacheWrite;
-		tokens.total += totalTokens > 0 ? totalTokens : input + output + cacheRead + cacheWrite;
-		if (isRecord(usage.cost)) costUsd += numberField(usage.cost, "total");
+		if (!isRecord(event)) return;
+		observeProviderEvent(provider, event);
+		if (event.type !== "message_end" || !isRecord(event.message) || event.message.role !== "assistant") return;
+		const usage = readMessageUsage(event.message);
+		if (usage === null) return;
+		if (Object.keys(usage.tokens).length > 0) {
+			measured = true;
+			for (const field of TOKEN_FIELDS) tokens[field] += usage.tokens[field] ?? 0;
+		}
+		costUsd += usage.costUsd ?? 0;
 	};
 
 	return {
@@ -85,7 +87,7 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 				consume(pending.replace(/\r$/u, ""));
 				pending = "";
 			}
-			return { measured, tokens: { ...tokens }, costUsd };
+			return { measured, tokens: { ...tokens }, costUsd, provider: structuredClone(provider) };
 		},
 	};
 }
@@ -101,6 +103,7 @@ export function addTokenStreamUsage(left: EvalTokenStreamUsage, right: EvalToken
 			cacheWrite: left.tokens.cacheWrite + right.tokens.cacheWrite,
 		},
 		costUsd: left.costUsd + right.costUsd,
+		provider: addProviderObservations(left.provider, right.provider),
 	};
 }
 
@@ -110,21 +113,188 @@ export function addTokenStreamUsage(left: EvalTokenStreamUsage, right: EvalToken
  * read an absence as a zero-cost run.
  */
 export function tokenMetricEntries(usage: EvalTokenStreamUsage): Record<string, number | boolean> {
-	if (!usage.measured) return { "tokens.measured": false };
 	return {
-		"tokens.measured": true,
-		"tokens.input": usage.tokens.input,
-		"tokens.output": usage.tokens.output,
-		"tokens.total": usage.tokens.total,
-		"tokens.cacheRead": usage.tokens.cacheRead,
-		"tokens.cacheWrite": usage.tokens.cacheWrite,
+		...providerMetricEntries(usage.provider),
+		"tokens.measured": usage.measured,
+		...(usage.measured
+			? {
+					"tokens.input": usage.tokens.input,
+					"tokens.output": usage.tokens.output,
+					"tokens.total": usage.tokens.total,
+					"tokens.cacheRead": usage.tokens.cacheRead,
+					"tokens.cacheWrite": usage.tokens.cacheWrite,
+				}
+			: {}),
 		...(usage.costUsd > 0 ? { "cost.usd": usage.costUsd } : {}),
 	};
 }
 
-function numberField(record: Record<string, unknown>, field: string): number {
-	const value = record[field];
-	return typeof value === "number" && Number.isFinite(value) ? value : 0;
+const TOKEN_FIELDS = ["input", "output", "total", "cacheRead", "cacheWrite"] as const;
+const STOP_REASONS = ["stop", "toolUse", "length", "error", "aborted", "other"] as const;
+const RETRY_PHASES = {
+	scheduled: "retryScheduled",
+	retrying: "retryStarted",
+	cancelled: "retryCancelled",
+	exhausted: "retryExhausted",
+	recovered: "retryRecovered",
+} as const;
+type StopReason = (typeof STOP_REASONS)[number];
+type RetryPhase = keyof typeof RETRY_PHASES;
+
+interface EvalProviderObservation {
+	measured: boolean;
+	stops: Record<StopReason, number>;
+	retries: Record<RetryPhase, number>;
+	errorUsageObservedCalls: number;
+	errorUsageUnobservedCalls: number;
+	errorUsageIncompleteCalls: number;
+	errorCostUnobservedCalls: number;
+	errorReasoningUnobservedCalls: number;
+	errorReasoningTokens: number | null;
+	errorTokens: Partial<EvalTokenMetricsV4>;
+	errorCostUsd: number | null;
+}
+
+interface MessageUsageObservation {
+	reasoningTokens: number | null;
+	tokens: Partial<EvalTokenMetricsV4>;
+	costUsd: number | null;
+	completeTokens: boolean;
+}
+
+function emptyProviderObservation(): EvalProviderObservation {
+	return {
+		measured: false,
+		stops: { stop: 0, toolUse: 0, length: 0, error: 0, aborted: 0, other: 0 },
+		retries: { scheduled: 0, retrying: 0, cancelled: 0, exhausted: 0, recovered: 0 },
+		errorUsageObservedCalls: 0,
+		errorUsageUnobservedCalls: 0,
+		errorUsageIncompleteCalls: 0,
+		errorCostUnobservedCalls: 0,
+		errorReasoningUnobservedCalls: 0,
+		errorReasoningTokens: null,
+		errorTokens: {},
+		errorCostUsd: null,
+	};
+}
+
+/** Same usage interpretation for inclusive totals and their known errored share. */
+function readMessageUsage(message: Record<string, unknown>): MessageUsageObservation | null {
+	if (!isRecord(message.usage)) return null;
+	const usage = message.usage;
+	const failed = message.stopReason === "error" || message.stopReason === "aborted";
+	const raw: Partial<EvalTokenMetricsV4> = {};
+	for (const field of TOKEN_FIELDS) {
+		const value = nonNegativeNumber(usage[field === "total" ? "totalTokens" : field]);
+		// Failed adapter objects can initialize each absent field to zero even
+		// when another field is reported. Keep only attributable positive facts.
+		if (value !== null && (!failed || value > 0)) raw[field] = value;
+	}
+	const cost = isRecord(usage.cost) ? nonNegativeNumber(usage.cost.total) : null;
+	// Zero prices are also initialized by adapters without pricing evidence.
+	const costUsd = cost !== null && cost > 0 ? cost : null;
+
+	// The compat adapter can synthesize the root reasoningTokens alias from
+	// thinking text without marking it estimated. Preserve explicit provider
+	// detail through the shared reader, but never promote that ambiguous alias.
+	// Normalized zero reasoning can also mean omitted provider detail.
+	const reportedReasoning = extractReasoningTokens({ ...usage, reasoningTokens: undefined });
+	const reasoningTokens = reportedReasoning !== null && reportedReasoning > 0 ? reportedReasoning : null;
+	const tokens = raw;
+	if (Object.keys(tokens).length === 0 && costUsd === null && reasoningTokens === null) return null;
+	const completeTokens = Object.keys(tokens).length === TOKEN_FIELDS.length;
+	if (Object.keys(tokens).length > 0 && !(tokens.total !== undefined && tokens.total > 0)) {
+		// This is a known subtotal when fields are missing; coverage stays incomplete.
+		tokens.total = (tokens.input ?? 0) + (tokens.output ?? 0) + (tokens.cacheRead ?? 0) + (tokens.cacheWrite ?? 0);
+	}
+	return { tokens, costUsd, completeTokens, reasoningTokens };
+}
+
+function observeProviderEvent(provider: EvalProviderObservation, event: Record<string, unknown>): void {
+	if (event.type === "retry_status" && isRecord(event.status)) {
+		const phase = event.status.phase;
+		if (typeof phase === "string" && Object.hasOwn(RETRY_PHASES, phase)) {
+			provider.measured = true;
+			provider.retries[phase as RetryPhase] += 1;
+		}
+		// Waiting frames are countdown ticks, not attempts. Attempt numbers may
+		// restart in another chain, so neither frames nor attempts are deduplicated.
+		return;
+	}
+	if (event.type !== "message_end" || !isRecord(event.message) || event.message.role !== "assistant") return;
+	const reason = event.message.stopReason;
+	if (typeof reason !== "string" || reason.length === 0 || reason === "pending") return;
+	provider.measured = true;
+	const stop = STOP_REASONS.includes(reason as StopReason) ? (reason as StopReason) : "other";
+	provider.stops[stop] += 1;
+	if (reason !== "error") return;
+	const usage = readMessageUsage(event.message);
+	if (usage === null) provider.errorUsageUnobservedCalls += 1;
+	else {
+		provider.errorUsageObservedCalls += 1;
+		for (const field of TOKEN_FIELDS) {
+			const value = usage.tokens[field];
+			if (value !== undefined) provider.errorTokens[field] = (provider.errorTokens[field] ?? 0) + value;
+		}
+	}
+	if (usage?.completeTokens !== true) provider.errorUsageIncompleteCalls += 1;
+	if (usage?.costUsd === undefined || usage.costUsd === null) provider.errorCostUnobservedCalls += 1;
+	else provider.errorCostUsd = (provider.errorCostUsd ?? 0) + usage.costUsd;
+	if (usage?.reasoningTokens === undefined || usage.reasoningTokens === null)
+		provider.errorReasoningUnobservedCalls += 1;
+	else provider.errorReasoningTokens = (provider.errorReasoningTokens ?? 0) + usage.reasoningTokens;
+}
+
+function addProviderObservations(
+	left: EvalProviderObservation,
+	right: EvalProviderObservation,
+): EvalProviderObservation {
+	const result = emptyProviderObservation();
+	result.measured = left.measured || right.measured;
+	for (const reason of STOP_REASONS) result.stops[reason] = left.stops[reason] + right.stops[reason];
+	for (const phase of Object.keys(RETRY_PHASES) as RetryPhase[])
+		result.retries[phase] = left.retries[phase] + right.retries[phase];
+	result.errorUsageObservedCalls = left.errorUsageObservedCalls + right.errorUsageObservedCalls;
+	result.errorUsageUnobservedCalls = left.errorUsageUnobservedCalls + right.errorUsageUnobservedCalls;
+	result.errorUsageIncompleteCalls = left.errorUsageIncompleteCalls + right.errorUsageIncompleteCalls;
+	result.errorCostUnobservedCalls = left.errorCostUnobservedCalls + right.errorCostUnobservedCalls;
+	result.errorReasoningUnobservedCalls = left.errorReasoningUnobservedCalls + right.errorReasoningUnobservedCalls;
+	if (left.errorReasoningTokens !== null || right.errorReasoningTokens !== null)
+		result.errorReasoningTokens = (left.errorReasoningTokens ?? 0) + (right.errorReasoningTokens ?? 0);
+	for (const field of TOKEN_FIELDS) {
+		if (left.errorTokens[field] !== undefined || right.errorTokens[field] !== undefined) {
+			result.errorTokens[field] = (left.errorTokens[field] ?? 0) + (right.errorTokens[field] ?? 0);
+		}
+	}
+	if (left.errorCostUsd !== null || right.errorCostUsd !== null)
+		result.errorCostUsd = (left.errorCostUsd ?? 0) + (right.errorCostUsd ?? 0);
+	return result;
+}
+
+/** Provider observations are independent of task success and receipt billing. */
+function providerMetricEntries(provider: EvalProviderObservation): Record<string, number | boolean> {
+	if (!provider.measured) return { "provider.measured": false };
+	const metrics: Record<string, number | boolean> = { "provider.measured": true };
+	for (const reason of STOP_REASONS) metrics[`provider.stopReason.${reason}`] = provider.stops[reason];
+	for (const phase of Object.keys(RETRY_PHASES) as RetryPhase[])
+		metrics[`provider.${RETRY_PHASES[phase]}`] = provider.retries[phase];
+	if (provider.stops.error === 0) return metrics;
+	metrics["provider.errorUsageObservedCalls"] = provider.errorUsageObservedCalls;
+	metrics["provider.errorUsageUnobservedCalls"] = provider.errorUsageUnobservedCalls;
+	metrics["provider.errorUsageIncompleteCalls"] = provider.errorUsageIncompleteCalls;
+	metrics["provider.errorCostUnobservedCalls"] = provider.errorCostUnobservedCalls;
+	metrics["provider.errorReasoningUnobservedCalls"] = provider.errorReasoningUnobservedCalls;
+	if (provider.errorReasoningTokens !== null) metrics["provider.errorReasoningTokens"] = provider.errorReasoningTokens;
+	for (const field of TOKEN_FIELDS) {
+		const value = provider.errorTokens[field];
+		if (value !== undefined) metrics[`provider.errorTokens.${field}`] = value;
+	}
+	if (provider.errorCostUsd !== null) metrics["provider.errorCostUsd"] = provider.errorCostUsd;
+	return metrics;
+}
+
+function nonNegativeNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/domains/eval/metrics/tracked.ts

@@ -47,6 +47,7 @@ export interface BuildEvalTrackedMetricsInput {
 }
 
 interface AssistantCall {
+	timestamp: string | null;
 	payload: Record<string, unknown>;
 	promptCache: Record<string, unknown> | null;
 	backend: Record<string, unknown> | null;
@@ -59,6 +60,28 @@ interface NumericReading {
 	source: EvalMetricSource;
 }
 
+/**
+ * The session ledger and stdout fold observe the same assistant calls on the
+ * native runner. Prefer the durable calls (including their richer timing and
+ * cache facts), with stdout as a fallback only when no durable calls exist.
+ * Keep non-call entries in either case: compaction is billed but not streamed.
+ * Counts are observations, NOT identities or evidence of matching coverage;
+ * partial/mixed ledgers cannot be unioned without a shared call identity.
+ */
+export function selectEvalLedgerEntries(
+	durable: ReadonlyArray<SessionEntry>,
+	stream: ReadonlyArray<SessionEntry>,
+): { entries: ReadonlyArray<SessionEntry>; source: "session" | "stream"; sessionCalls: number; streamCalls: number } {
+	const sessionCalls = assistantCalls(durable).length;
+	const streamCalls = assistantCalls(stream).length;
+	return {
+		entries: sessionCalls > 0 ? durable : [...durable, ...stream],
+		source: sessionCalls > 0 ? "session" : "stream",
+		sessionCalls,
+		streamCalls,
+	};
+}
+
 /** Read every isolated session ledger and its prompt manifest before cleanup. */
 export async function readEvalLedgerSnapshot(stateDir: string): Promise<EvalLedgerSnapshot> {
 	const refs = await listSessionLedgerRefs(stateDir);
@@ -175,7 +198,7 @@ export function emptyEvalTrackedMetrics(source: EvalMetricSource = "estimated"):
 		reasoningTokens: { value: null, source },
 		toolCalls: zero(),
 		toolErrors: zero(),
-		ttftMsFirstCall: zero(),
+		ttftMsFirstCall: { value: null, source: "estimated" },
 		wallClockMs: zero(),
 		contextTokensAtEnd: zero(),
 		compactions: zero(),
@@ -192,6 +215,7 @@ function assistantCalls(entries: ReadonlyArray<SessionEntry>): AssistantCall[] {
 		if (promptCache === null && timing === null && usage === null) return [];
 		return [
 			{
+				timestamp: entry.payload.timestampEstimated === true ? null : entry.timestamp,
 				payload: entry.payload,
 				promptCache,
 				backend: promptCache === null ? null : recordField(promptCache, "backend"),
@@ -260,10 +284,22 @@ function reasoningMetric(
 	return measured ? { value: total, source: "ledger" } : { value: null, source: "estimated" };
 }
 
-function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNumber {
-	const first = calls[0];
+function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNullableNumber {
+	// Session directories are read by name, not by call time. Select without
+	// reordering the evidence or borrowing a later call's available timing.
+	let first: AssistantCall | undefined;
+	let firstAt = Number.POSITIVE_INFINITY;
+	for (const call of calls) {
+		if (call.timestamp === null) return { value: null, source: "estimated" };
+		const at = Date.parse(call.timestamp);
+		if (!Number.isFinite(at)) return { value: null, source: "estimated" };
+		if (at < firstAt) {
+			first = call;
+			firstAt = at;
+		}
+	}
 	const value = first?.timing === null || first?.timing === undefined ? null : nonNegativeNumber(first.timing.ttftMs);
-	return value === null ? { value: 0, source: "estimated" } : { value, source: "ledger" };
+	return value === null ? { value: null, source: "estimated" } : { value, source: "ledger" };
 }
 
 function wallClockMetric(receipt: RunReceipt | null, fallback: number): EvalSourcedNumber {

package/src/domains/eval/runners/clio-run.ts

@@ -149,7 +149,7 @@ interface BehavioralToolTerminal {
  * bounded counters against the suite's explicit public allowlist and decoy
  * list; they never become behavioral fact values or evidence excerpts.
  */
-function toolBehaviorMetricEntriesFromJsonl(
+export function toolBehaviorMetricEntriesFromJsonl(
 	stdout: string,
 	cwd: string,
 	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
@@ -201,16 +201,19 @@ function toolBehaviorMetricEntriesFromJsonl(
 
 	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
 	const calls = new Map<string, number>();
+	const succeeded = new Map<string, number>();
 	const blocked = new Map<string, number>();
 	for (const terminal of terminals) {
 		const tool = metricToolName(terminal.tool);
 		calls.set(tool, (calls.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "ok") succeeded.set(tool, (succeeded.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
 	}
-	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...blocked.keys()]);
+	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...succeeded.keys(), ...blocked.keys()]);
 	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
 	for (const tool of [...namedTools].sort()) {
 		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
+		entries[`tools.succeeded.${tool}`] = succeeded.get(tool) ?? 0;
 		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
 	}
 	if (readObservation !== undefined) {

package/src/domains/eval/schema/suite.ts

@@ -13,6 +13,33 @@ export const CANONICAL_METRICS = [
 	// are absent in that case rather than zero, so a gate on cost can require
 	// measurement instead of reading silence as free.
 	"tokens.measured",
+	// Observed stdout terminal/retry facts, independent of task success. Opaque
+	// runs carry measured:false and no counters. Errored usage stays in inclusive
+	// tokens.* totals; errorTokens/errorCostUsd are known subtotals with coverage.
+	"provider.measured",
+	"provider.stopReason.stop",
+	"provider.stopReason.toolUse",
+	"provider.stopReason.length",
+	"provider.stopReason.error",
+	"provider.stopReason.aborted",
+	"provider.stopReason.other",
+	"provider.retryScheduled",
+	"provider.retryStarted",
+	"provider.retryCancelled",
+	"provider.retryExhausted",
+	"provider.retryRecovered",
+	"provider.errorUsageObservedCalls",
+	"provider.errorUsageUnobservedCalls",
+	"provider.errorUsageIncompleteCalls",
+	"provider.errorCostUnobservedCalls",
+	"provider.errorReasoningUnobservedCalls",
+	"provider.errorReasoningTokens",
+	"provider.errorTokens.input",
+	"provider.errorTokens.output",
+	"provider.errorTokens.total",
+	"provider.errorTokens.cacheRead",
+	"provider.errorTokens.cacheWrite",
+	"provider.errorCostUsd",
 	"latency.wallMs",
 	"latency.modelMs",
 	"tools.totalCalls",
@@ -22,6 +49,7 @@ export const CANONICAL_METRICS = [
 	"tools.read.outsideAllowed",
 	"tools.read.decoyHits",
 	"tools.calls.dispatch",
+	"tools.succeeded.dispatch",
 	"tools.blocked.bash",
 	"claims.unsupported",
 	"completion.reported",

package/src/domains/eval/schema/verdict.ts

@@ -39,7 +39,7 @@ export interface EvalTrackedMetricsV1 {
 	reasoningTokens: EvalSourcedNullableNumber;
 	toolCalls: EvalSourcedNumber;
 	toolErrors: EvalSourcedNumber;
-	ttftMsFirstCall: EvalSourcedNumber;
+	ttftMsFirstCall: EvalSourcedNullableNumber;
 	wallClockMs: EvalSourcedNumber;
 	contextTokensAtEnd: EvalSourcedNumber;
 	compactions: EvalSourcedNumber;
@@ -125,7 +125,7 @@ function parseTrackedMetrics(value: unknown, source: string): EvalTrackedMetrics
 		reasoningTokens: readSourcedNullableNumber(record.reasoningTokens, `${source}.reasoningTokens`),
 		toolCalls: readSourcedNumber(record.toolCalls, `${source}.toolCalls`),
 		toolErrors: readSourcedNumber(record.toolErrors, `${source}.toolErrors`),
-		ttftMsFirstCall: readSourcedNumber(record.ttftMsFirstCall, `${source}.ttftMsFirstCall`),
+		ttftMsFirstCall: readSourcedNullableNumber(record.ttftMsFirstCall, `${source}.ttftMsFirstCall`),
 		wallClockMs: readSourcedNumber(record.wallClockMs, `${source}.wallClockMs`),
 		contextTokensAtEnd: readSourcedNumber(record.contextTokensAtEnd, `${source}.contextTokensAtEnd`),
 		compactions: readSourcedNumber(record.compactions, `${source}.compactions`),

package/src/domains/eval/store.ts

@@ -60,9 +60,16 @@ export function evalRoot(dataDir: string): string {
 	return join(dataDir, "evals");
 }
 
+/**
+ * `skills-eval` is the only writer of this legacy version-1 shape; it nests
+ * under its own subdirectory so a legacy id that collides with a `clio-coder
+ * eval` run's version-4 id (evalArtifactPathV4, artifacts/store.ts) cannot
+ * clobber that artifact, and so `eval inventory`'s top-level readdir of
+ * evalRoot() never trips over a shape it does not parse.
+ */
 export function evalArtifactPath(dataDir: string, evalId: string): string {
 	assertSafeId(evalId, "eval");
-	return join(evalRoot(dataDir), `${evalId}.json`);
+	return join(evalRoot(dataDir), "skills-eval", `${evalId}.json`);
 }
 
 /**

package/src/domains/eval/suites/resolve.ts

@@ -38,7 +38,19 @@ function resolveTargets(targets: ReadonlyArray<EvalSuiteTargetV2>, options: Reso
 	const filtered =
 		options.target === undefined ? [...targets] : targets.filter((target) => target.id === options.target);
 	const selected =
-		filtered.length > 0 ? filtered : options.target === undefined ? [...targets] : [{ id: options.target }];
+		filtered.length > 0
+			? filtered
+			: options.target === undefined
+				? [...targets]
+				: [
+						{
+							id: options.target,
+							// A one-row suite still owns its thinking dimension when the CLI
+							// substitutes the target/model pair. Dropping it silently changes
+							// an off baseline into the target's configured default.
+							...(targets.length === 1 && targets[0]?.thinking !== undefined ? { thinking: targets[0].thinking } : {}),
+						},
+					];
 	return selected.map((target) => ({
 		...target,
 		...(options.model === undefined ? {} : { model: options.model }),

package/src/domains/eval/suites/run.ts

@@ -18,7 +18,12 @@ import {
 import { wallTimeMetric } from "../metrics/latency.js";
 import { tokenAccountingFrom } from "../metrics/tokens.js";
 import { zeroToolCallMetrics } from "../metrics/tool-calls.js";
-import { buildEvalTrackedMetrics, emptyEvalTrackedMetrics, readEvalLedgerSnapshot } from "../metrics/tracked.js";
+import {
+	buildEvalTrackedMetrics,
+	emptyEvalTrackedMetrics,
+	readEvalLedgerSnapshot,
+	selectEvalLedgerEntries,
+} from "../metrics/tracked.js";
 import {
 	type EvalServingObservation,
 	evalClioProvenance,
@@ -234,11 +239,27 @@ async function runMatrixItem(
 			},
 		};
 		const snapshot = await readEvalLedgerSnapshot(stateDir);
-		const ledgerEntries = [...snapshot.entries, ...(runner.ledgerEntries ?? [])];
+		const selected = selectEvalLedgerEntries(snapshot.entries, runner.ledgerEntries ?? []);
+		result.artifacts.trackedMetricSources = JSON.stringify({
+			assistantCalls: selected.source,
+			sessionCalls: selected.sessionCalls,
+			streamCalls: selected.streamCalls,
+			compactions: "session",
+		});
+		if (selected.sessionCalls > 0 && selected.streamCalls > 0) {
+			// Preserve the unmerged evidence for opaque runners too. The native
+			// runner already publishes this artifact before stdout truncation.
+			result.artifacts.callLedger ??= JSON.stringify(runner.ledgerEntries);
+			if (selected.sessionCalls !== selected.streamCalls || task.runner.kind === "external-command") {
+				result.artifacts.trackedMetricWarning =
+					"Session and stream call coverage may differ; tracked metrics use session calls, not a reconciled union. " +
+					"Inspect callLedger before treating these counts as complete: no shared call identity proves overlap.";
+			}
+		}
 		result.verdict = adaptSuiteV2ResultToVerdictV1(
 			result,
 			buildEvalTrackedMetrics({
-				ledgerEntries,
+				ledgerEntries: selected.entries,
 				receipt: receipt ?? null,
 				fallbackWallClockMs: runner.wallTimeMs,
 			}),