@iowarp/clio-coder 0.4.2 → 0.4.4

package/src/domains/eval/metrics/token-stream.ts

@@ -1,14 +1,17 @@
+import { extractReasoningTokens } from "../../session/context-accounting.js";
 import type { EvalTokenMetricsV4 } from "../schema/artifact.js";
 
 export interface EvalTokenStreamUsage {
 	/**
 	 * True when the stream carried at least one completed assistant message
-	 * with provider usage. False means this runner observed no token
+	 * with observed token facts. False means this runner observed no token
 	 * accounting at all, which is not the same as a run that cost nothing.
 	 */
 	measured: boolean;
+	/** Known token subtotals; missing failed-call coverage is reported separately. */
 	tokens: EvalTokenMetricsV4;
 	costUsd: number;
+	provider: EvalProviderObservation;
 }
 
 export interface EvalTokenUsageFold {
@@ -21,6 +24,7 @@ export const UNMEASURED_TOKEN_USAGE: EvalTokenStreamUsage = {
 	measured: false,
 	tokens: { input: 0, output: 0, total: 0, cacheRead: 0, cacheWrite: 0 },
 	costUsd: 0,
+	provider: emptyProviderObservation(),
 };
 
 /**
@@ -30,8 +34,12 @@ export const UNMEASURED_TOKEN_USAGE: EvalTokenStreamUsage = {
  * carrying a completed message's usage exactly once; `turn_end` republishes
  * the same assistant message and `agent_end` republishes its segment's
  * summary, so counting those too would multiply a run's reported cost. Every
- * message counts, because a headless turn spans several agent segments and the
- * run's cost is their sum.
+ * reported call counts, including known errored-call spend: a headless turn
+ * spans several segments and its cost is their sum. Provider facts and the
+ * errored known share are folded here too, from the same events before truncation.
+ * Failed all-zero usage is ambiguous (adapters synthesize it), so it cannot
+ * establish measured usage. Unknown failed spend is retained as missing coverage,
+ * never subtracted from inclusive totals or used to change task pass policy.
  *
  * Folding as the stream arrives is what makes the count trustworthy: the
  * operator-facing stdout artifact keeps only a bounded head and tail, so a
@@ -42,6 +50,7 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 	let costUsd = 0;
 	let measured = false;
 	let pending = "";
+	const provider = emptyProviderObservation();
 
 	const consume = (line: string): void => {
 		if (line.trim().length === 0) return;
@@ -51,23 +60,16 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 		} catch {
 			return;
 		}
-		if (!isRecord(event) || event.type !== "message_end") return;
-		const message = isRecord(event.message) ? event.message : undefined;
-		if (message === undefined || message.role !== "assistant") return;
-		const usage = isRecord(message.usage) ? message.usage : undefined;
-		if (usage === undefined) return;
-		measured = true;
-		const input = numberField(usage, "input");
-		const output = numberField(usage, "output");
-		const cacheRead = numberField(usage, "cacheRead");
-		const cacheWrite = numberField(usage, "cacheWrite");
-		const totalTokens = numberField(usage, "totalTokens");
-		tokens.input += input;
-		tokens.output += output;
-		tokens.cacheRead += cacheRead;
-		tokens.cacheWrite += cacheWrite;
-		tokens.total += totalTokens > 0 ? totalTokens : input + output + cacheRead + cacheWrite;
-		if (isRecord(usage.cost)) costUsd += numberField(usage.cost, "total");
+		if (!isRecord(event)) return;
+		observeProviderEvent(provider, event);
+		if (event.type !== "message_end" || !isRecord(event.message) || event.message.role !== "assistant") return;
+		const usage = readMessageUsage(event.message);
+		if (usage === null) return;
+		if (Object.keys(usage.tokens).length > 0) {
+			measured = true;
+			for (const field of TOKEN_FIELDS) tokens[field] += usage.tokens[field] ?? 0;
+		}
+		costUsd += usage.costUsd ?? 0;
 	};
 
 	return {
@@ -85,7 +87,7 @@ export function createTokenUsageFold(): EvalTokenUsageFold {
 				consume(pending.replace(/\r$/u, ""));
 				pending = "";
 			}
-			return { measured, tokens: { ...tokens }, costUsd };
+			return { measured, tokens: { ...tokens }, costUsd, provider: structuredClone(provider) };
 		},
 	};
 }
@@ -101,6 +103,7 @@ export function addTokenStreamUsage(left: EvalTokenStreamUsage, right: EvalToken
 			cacheWrite: left.tokens.cacheWrite + right.tokens.cacheWrite,
 		},
 		costUsd: left.costUsd + right.costUsd,
+		provider: addProviderObservations(left.provider, right.provider),
 	};
 }
 
@@ -110,21 +113,188 @@ export function addTokenStreamUsage(left: EvalTokenStreamUsage, right: EvalToken
  * read an absence as a zero-cost run.
  */
 export function tokenMetricEntries(usage: EvalTokenStreamUsage): Record<string, number | boolean> {
-	if (!usage.measured) return { "tokens.measured": false };
 	return {
-		"tokens.measured": true,
-		"tokens.input": usage.tokens.input,
-		"tokens.output": usage.tokens.output,
-		"tokens.total": usage.tokens.total,
-		"tokens.cacheRead": usage.tokens.cacheRead,
-		"tokens.cacheWrite": usage.tokens.cacheWrite,
+		...providerMetricEntries(usage.provider),
+		"tokens.measured": usage.measured,
+		...(usage.measured
+			? {
+					"tokens.input": usage.tokens.input,
+					"tokens.output": usage.tokens.output,
+					"tokens.total": usage.tokens.total,
+					"tokens.cacheRead": usage.tokens.cacheRead,
+					"tokens.cacheWrite": usage.tokens.cacheWrite,
+				}
+			: {}),
 		...(usage.costUsd > 0 ? { "cost.usd": usage.costUsd } : {}),
 	};
 }
 
-function numberField(record: Record<string, unknown>, field: string): number {
-	const value = record[field];
-	return typeof value === "number" && Number.isFinite(value) ? value : 0;
+const TOKEN_FIELDS = ["input", "output", "total", "cacheRead", "cacheWrite"] as const;
+const STOP_REASONS = ["stop", "toolUse", "length", "error", "aborted", "other"] as const;
+const RETRY_PHASES = {
+	scheduled: "retryScheduled",
+	retrying: "retryStarted",
+	cancelled: "retryCancelled",
+	exhausted: "retryExhausted",
+	recovered: "retryRecovered",
+} as const;
+type StopReason = (typeof STOP_REASONS)[number];
+type RetryPhase = keyof typeof RETRY_PHASES;
+
+interface EvalProviderObservation {
+	measured: boolean;
+	stops: Record<StopReason, number>;
+	retries: Record<RetryPhase, number>;
+	errorUsageObservedCalls: number;
+	errorUsageUnobservedCalls: number;
+	errorUsageIncompleteCalls: number;
+	errorCostUnobservedCalls: number;
+	errorReasoningUnobservedCalls: number;
+	errorReasoningTokens: number | null;
+	errorTokens: Partial<EvalTokenMetricsV4>;
+	errorCostUsd: number | null;
+}
+
+interface MessageUsageObservation {
+	reasoningTokens: number | null;
+	tokens: Partial<EvalTokenMetricsV4>;
+	costUsd: number | null;
+	completeTokens: boolean;
+}
+
+function emptyProviderObservation(): EvalProviderObservation {
+	return {
+		measured: false,
+		stops: { stop: 0, toolUse: 0, length: 0, error: 0, aborted: 0, other: 0 },
+		retries: { scheduled: 0, retrying: 0, cancelled: 0, exhausted: 0, recovered: 0 },
+		errorUsageObservedCalls: 0,
+		errorUsageUnobservedCalls: 0,
+		errorUsageIncompleteCalls: 0,
+		errorCostUnobservedCalls: 0,
+		errorReasoningUnobservedCalls: 0,
+		errorReasoningTokens: null,
+		errorTokens: {},
+		errorCostUsd: null,
+	};
+}
+
+/** Same usage interpretation for inclusive totals and their known errored share. */
+function readMessageUsage(message: Record<string, unknown>): MessageUsageObservation | null {
+	if (!isRecord(message.usage)) return null;
+	const usage = message.usage;
+	const failed = message.stopReason === "error" || message.stopReason === "aborted";
+	const raw: Partial<EvalTokenMetricsV4> = {};
+	for (const field of TOKEN_FIELDS) {
+		const value = nonNegativeNumber(usage[field === "total" ? "totalTokens" : field]);
+		// Failed adapter objects can initialize each absent field to zero even
+		// when another field is reported. Keep only attributable positive facts.
+		if (value !== null && (!failed || value > 0)) raw[field] = value;
+	}
+	const cost = isRecord(usage.cost) ? nonNegativeNumber(usage.cost.total) : null;
+	// Zero prices are also initialized by adapters without pricing evidence.
+	const costUsd = cost !== null && cost > 0 ? cost : null;
+
+	// The compat adapter can synthesize the root reasoningTokens alias from
+	// thinking text without marking it estimated. Preserve explicit provider
+	// detail through the shared reader, but never promote that ambiguous alias.
+	// Normalized zero reasoning can also mean omitted provider detail.
+	const reportedReasoning = extractReasoningTokens({ ...usage, reasoningTokens: undefined });
+	const reasoningTokens = reportedReasoning !== null && reportedReasoning > 0 ? reportedReasoning : null;
+	const tokens = raw;
+	if (Object.keys(tokens).length === 0 && costUsd === null && reasoningTokens === null) return null;
+	const completeTokens = Object.keys(tokens).length === TOKEN_FIELDS.length;
+	if (Object.keys(tokens).length > 0 && !(tokens.total !== undefined && tokens.total > 0)) {
+		// This is a known subtotal when fields are missing; coverage stays incomplete.
+		tokens.total = (tokens.input ?? 0) + (tokens.output ?? 0) + (tokens.cacheRead ?? 0) + (tokens.cacheWrite ?? 0);
+	}
+	return { tokens, costUsd, completeTokens, reasoningTokens };
+}
+
+function observeProviderEvent(provider: EvalProviderObservation, event: Record<string, unknown>): void {
+	if (event.type === "retry_status" && isRecord(event.status)) {
+		const phase = event.status.phase;
+		if (typeof phase === "string" && Object.hasOwn(RETRY_PHASES, phase)) {
+			provider.measured = true;
+			provider.retries[phase as RetryPhase] += 1;
+		}
+		// Waiting frames are countdown ticks, not attempts. Attempt numbers may
+		// restart in another chain, so neither frames nor attempts are deduplicated.
+		return;
+	}
+	if (event.type !== "message_end" || !isRecord(event.message) || event.message.role !== "assistant") return;
+	const reason = event.message.stopReason;
+	if (typeof reason !== "string" || reason.length === 0 || reason === "pending") return;
+	provider.measured = true;
+	const stop = STOP_REASONS.includes(reason as StopReason) ? (reason as StopReason) : "other";
+	provider.stops[stop] += 1;
+	if (reason !== "error") return;
+	const usage = readMessageUsage(event.message);
+	if (usage === null) provider.errorUsageUnobservedCalls += 1;
+	else {
+		provider.errorUsageObservedCalls += 1;
+		for (const field of TOKEN_FIELDS) {
+			const value = usage.tokens[field];
+			if (value !== undefined) provider.errorTokens[field] = (provider.errorTokens[field] ?? 0) + value;
+		}
+	}
+	if (usage?.completeTokens !== true) provider.errorUsageIncompleteCalls += 1;
+	if (usage?.costUsd === undefined || usage.costUsd === null) provider.errorCostUnobservedCalls += 1;
+	else provider.errorCostUsd = (provider.errorCostUsd ?? 0) + usage.costUsd;
+	if (usage?.reasoningTokens === undefined || usage.reasoningTokens === null)
+		provider.errorReasoningUnobservedCalls += 1;
+	else provider.errorReasoningTokens = (provider.errorReasoningTokens ?? 0) + usage.reasoningTokens;
+}
+
+function addProviderObservations(
+	left: EvalProviderObservation,
+	right: EvalProviderObservation,
+): EvalProviderObservation {
+	const result = emptyProviderObservation();
+	result.measured = left.measured || right.measured;
+	for (const reason of STOP_REASONS) result.stops[reason] = left.stops[reason] + right.stops[reason];
+	for (const phase of Object.keys(RETRY_PHASES) as RetryPhase[])
+		result.retries[phase] = left.retries[phase] + right.retries[phase];
+	result.errorUsageObservedCalls = left.errorUsageObservedCalls + right.errorUsageObservedCalls;
+	result.errorUsageUnobservedCalls = left.errorUsageUnobservedCalls + right.errorUsageUnobservedCalls;
+	result.errorUsageIncompleteCalls = left.errorUsageIncompleteCalls + right.errorUsageIncompleteCalls;
+	result.errorCostUnobservedCalls = left.errorCostUnobservedCalls + right.errorCostUnobservedCalls;
+	result.errorReasoningUnobservedCalls = left.errorReasoningUnobservedCalls + right.errorReasoningUnobservedCalls;
+	if (left.errorReasoningTokens !== null || right.errorReasoningTokens !== null)
+		result.errorReasoningTokens = (left.errorReasoningTokens ?? 0) + (right.errorReasoningTokens ?? 0);
+	for (const field of TOKEN_FIELDS) {
+		if (left.errorTokens[field] !== undefined || right.errorTokens[field] !== undefined) {
+			result.errorTokens[field] = (left.errorTokens[field] ?? 0) + (right.errorTokens[field] ?? 0);
+		}
+	}
+	if (left.errorCostUsd !== null || right.errorCostUsd !== null)
+		result.errorCostUsd = (left.errorCostUsd ?? 0) + (right.errorCostUsd ?? 0);
+	return result;
+}
+
+/** Provider observations are independent of task success and receipt billing. */
+function providerMetricEntries(provider: EvalProviderObservation): Record<string, number | boolean> {
+	if (!provider.measured) return { "provider.measured": false };
+	const metrics: Record<string, number | boolean> = { "provider.measured": true };
+	for (const reason of STOP_REASONS) metrics[`provider.stopReason.${reason}`] = provider.stops[reason];
+	for (const phase of Object.keys(RETRY_PHASES) as RetryPhase[])
+		metrics[`provider.${RETRY_PHASES[phase]}`] = provider.retries[phase];
+	if (provider.stops.error === 0) return metrics;
+	metrics["provider.errorUsageObservedCalls"] = provider.errorUsageObservedCalls;
+	metrics["provider.errorUsageUnobservedCalls"] = provider.errorUsageUnobservedCalls;
+	metrics["provider.errorUsageIncompleteCalls"] = provider.errorUsageIncompleteCalls;
+	metrics["provider.errorCostUnobservedCalls"] = provider.errorCostUnobservedCalls;
+	metrics["provider.errorReasoningUnobservedCalls"] = provider.errorReasoningUnobservedCalls;
+	if (provider.errorReasoningTokens !== null) metrics["provider.errorReasoningTokens"] = provider.errorReasoningTokens;
+	for (const field of TOKEN_FIELDS) {
+		const value = provider.errorTokens[field];
+		if (value !== undefined) metrics[`provider.errorTokens.${field}`] = value;
+	}
+	if (provider.errorCostUsd !== null) metrics["provider.errorCostUsd"] = provider.errorCostUsd;
+	return metrics;
+}
+
+function nonNegativeNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/domains/eval/metrics/tracked.ts

@@ -47,6 +47,7 @@ export interface BuildEvalTrackedMetricsInput {
 }
 
 interface AssistantCall {
+	timestamp: string | null;
 	payload: Record<string, unknown>;
 	promptCache: Record<string, unknown> | null;
 	backend: Record<string, unknown> | null;
@@ -59,6 +60,28 @@ interface NumericReading {
 	source: EvalMetricSource;
 }
 
+/**
+ * The session ledger and stdout fold observe the same assistant calls on the
+ * native runner. Prefer the durable calls (including their richer timing and
+ * cache facts), with stdout as a fallback only when no durable calls exist.
+ * Keep non-call entries in either case: compaction is billed but not streamed.
+ * Counts are observations, NOT identities or evidence of matching coverage;
+ * partial/mixed ledgers cannot be unioned without a shared call identity.
+ */
+export function selectEvalLedgerEntries(
+	durable: ReadonlyArray<SessionEntry>,
+	stream: ReadonlyArray<SessionEntry>,
+): { entries: ReadonlyArray<SessionEntry>; source: "session" | "stream"; sessionCalls: number; streamCalls: number } {
+	const sessionCalls = assistantCalls(durable).length;
+	const streamCalls = assistantCalls(stream).length;
+	return {
+		entries: sessionCalls > 0 ? durable : [...durable, ...stream],
+		source: sessionCalls > 0 ? "session" : "stream",
+		sessionCalls,
+		streamCalls,
+	};
+}
+
 /** Read every isolated session ledger and its prompt manifest before cleanup. */
 export async function readEvalLedgerSnapshot(stateDir: string): Promise<EvalLedgerSnapshot> {
 	const refs = await listSessionLedgerRefs(stateDir);
@@ -175,7 +198,7 @@ export function emptyEvalTrackedMetrics(source: EvalMetricSource = "estimated"):
 		reasoningTokens: { value: null, source },
 		toolCalls: zero(),
 		toolErrors: zero(),
-		ttftMsFirstCall: zero(),
+		ttftMsFirstCall: { value: null, source: "estimated" },
 		wallClockMs: zero(),
 		contextTokensAtEnd: zero(),
 		compactions: zero(),
@@ -192,6 +215,7 @@ function assistantCalls(entries: ReadonlyArray<SessionEntry>): AssistantCall[] {
 		if (promptCache === null && timing === null && usage === null) return [];
 		return [
 			{
+				timestamp: entry.payload.timestampEstimated === true ? null : entry.timestamp,
 				payload: entry.payload,
 				promptCache,
 				backend: promptCache === null ? null : recordField(promptCache, "backend"),
@@ -260,10 +284,22 @@ function reasoningMetric(
 	return measured ? { value: total, source: "ledger" } : { value: null, source: "estimated" };
 }
 
-function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNumber {
-	const first = calls[0];
+function firstCallTtft(calls: ReadonlyArray<AssistantCall>): EvalSourcedNullableNumber {
+	// Session directories are read by name, not by call time. Select without
+	// reordering the evidence or borrowing a later call's available timing.
+	let first: AssistantCall | undefined;
+	let firstAt = Number.POSITIVE_INFINITY;
+	for (const call of calls) {
+		if (call.timestamp === null) return { value: null, source: "estimated" };
+		const at = Date.parse(call.timestamp);
+		if (!Number.isFinite(at)) return { value: null, source: "estimated" };
+		if (at < firstAt) {
+			first = call;
+			firstAt = at;
+		}
+	}
 	const value = first?.timing === null || first?.timing === undefined ? null : nonNegativeNumber(first.timing.ttftMs);
-	return value === null ? { value: 0, source: "estimated" } : { value, source: "ledger" };
+	return value === null ? { value: null, source: "estimated" } : { value, source: "ledger" };
 }
 
 function wallClockMetric(receipt: RunReceipt | null, fallback: number): EvalSourcedNumber {

package/src/domains/eval/runners/clio-run.ts

@@ -1,6 +1,5 @@
 import { isAbsolute, relative, resolve, sep } from "node:path";
 import { normalizeClioCoderEventRecord } from "../../../core/naming-events.js";
-import { shellQuote } from "../../../core/shell-quote.js";
 import { clioStateDir } from "../../../core/xdg.js";
 import {
 	evidenceMetricsFromReceipt,
@@ -22,19 +21,23 @@ export async function runClioRunRunner(
 	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
 ): Promise<EvalRunnerOutput> {
 	const prompt = runner.prompt ?? "";
-	const args = [
-		shellQuote(clioEntry),
+	// Spawned as an argv, not a shell line: the task timeout has to reach the
+	// Clio process itself so the receipt it seals and the exit code this
+	// runner reports describe the same ending. See runShellCommand.
+	const args: [string, ...string[]] = [
+		process.execPath,
+		clioEntry,
 		"run",
 		"--json",
-		...(runner.agent === undefined ? [] : ["--agent", shellQuote(runner.agent)]),
+		...(runner.agent === undefined ? [] : ["--agent", runner.agent]),
 		"--target",
-		shellQuote(target.id),
-		...(target.model === undefined ? [] : ["--model", shellQuote(target.model)]),
-		...(target.thinking === undefined ? [] : ["--thinking", shellQuote(target.thinking)]),
+		target.id,
+		...(target.model === undefined ? [] : ["--model", target.model]),
+		...(target.thinking === undefined ? [] : ["--thinking", target.thinking]),
 		...(runner.autonomy === undefined ? [] : ["--autonomy", runner.autonomy]),
-		shellQuote(prompt),
+		prompt,
 	];
-	const result = await runShellCommand(`${process.execPath} ${args.join(" ")}`, cwd, runner.timeoutMs ?? timeoutMs, env);
+	const result = await runShellCommand(args, cwd, runner.timeoutMs ?? timeoutMs, env);
 	// Usage is folded from the live stream, not from the bounded stdout
 	// artifact: a verbose run's `message_end` events do not survive truncation.
 	const tokens = result.usage;
@@ -149,7 +152,7 @@ interface BehavioralToolTerminal {
  * bounded counters against the suite's explicit public allowlist and decoy
  * list; they never become behavioral fact values or evidence excerpts.
  */
-function toolBehaviorMetricEntriesFromJsonl(
+export function toolBehaviorMetricEntriesFromJsonl(
 	stdout: string,
 	cwd: string,
 	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
@@ -201,16 +204,19 @@ function toolBehaviorMetricEntriesFromJsonl(
 
 	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
 	const calls = new Map<string, number>();
+	const succeeded = new Map<string, number>();
 	const blocked = new Map<string, number>();
 	for (const terminal of terminals) {
 		const tool = metricToolName(terminal.tool);
 		calls.set(tool, (calls.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "ok") succeeded.set(tool, (succeeded.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
 	}
-	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...blocked.keys()]);
+	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...succeeded.keys(), ...blocked.keys()]);
 	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
 	for (const tool of [...namedTools].sort()) {
 		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
+		entries[`tools.succeeded.${tool}`] = succeeded.get(tool) ?? 0;
 		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
 	}
 	if (readObservation !== undefined) {

package/src/domains/eval/runners/context-index.ts

@@ -1,4 +1,3 @@
-import { shellQuote } from "../../../core/shell-quote.js";
 import type { EvalSuiteTargetV2 } from "../schema/suite.js";
 import { type EvalRunnerOutput, runShellCommand } from "./external-command.js";
 
@@ -9,12 +8,8 @@ export async function runContextIndexRunner(
 	target: EvalSuiteTargetV2,
 	env?: NodeJS.ProcessEnv,
 ): Promise<EvalRunnerOutput> {
-	const result = await runShellCommand(
-		`${shellQuote(process.execPath)} ${shellQuote(clioEntry)} context index --json`,
-		cwd,
-		timeoutMs,
-		env,
-	);
+	const args: [string, ...string[]] = [process.execPath, clioEntry, "context", "index", "--json"];
+	const result = await runShellCommand(args, cwd, timeoutMs, env);
 	const parsed = parseContextIndexOutput(result.stdout);
 	return {
 		assignmentId: null,

package/src/domains/eval/runners/context-init.ts

@@ -1,6 +1,5 @@
 import { existsSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { shellQuote } from "../../../core/shell-quote.js";
 import type { EvalRunnerV2, EvalSuiteTargetV2 } from "../schema/suite.js";
 import { type EvalRunnerOutput, runShellCommand } from "./external-command.js";
 
@@ -13,7 +12,7 @@ export async function runContextInitRunner(
 	env?: NodeJS.ProcessEnv,
 ): Promise<EvalRunnerOutput> {
 	const extraArgs = runner.args ?? [];
-	const command = [
+	const args: [string, ...string[]] = [
 		process.execPath,
 		clioEntry,
 		"context",
@@ -25,10 +24,8 @@ export async function runContextInitRunner(
 		...(target.model === undefined ? [] : ["--model", target.model]),
 		...(target.thinking === undefined ? [] : ["--thinking", target.thinking]),
 		...extraArgs,
-	]
-		.map(shellQuote)
-		.join(" ");
-	const result = await runShellCommand(command, cwd, runner.timeoutMs ?? timeoutMs, env);
+	];
+	const result = await runShellCommand(args, cwd, runner.timeoutMs ?? timeoutMs, env);
 	const payload = parseInitPayload(result.stdout);
 	const candidateGeneration = recordField(payload, "generation");
 	const generation = isValidGenerationPayload(payload, candidateGeneration) ? candidateGeneration : null;

package/src/domains/eval/runners/external-command.ts

@@ -65,6 +65,11 @@ const OUTPUT_HEAD_LIMIT = 20_000;
 const OUTPUT_TRUNCATION_MARKER = "\n[output middle truncated; tail preserved]\n";
 const METRIC_JSONL_LIMIT = 256_000;
 const METRIC_JSONL_LINE_LIMIT = 64_000;
+// After the deadline's SIGTERM, how long a run gets to end on its own terms
+// before SIGKILL. Clio seals the interrupted run's receipt inside its
+// coordinated shutdown, which budgets 500ms per hook across several hooks; a
+// one-second grace killed it mid-seal and the timed-out run left no receipt.
+const TIMEOUT_KILL_GRACE_MS = 5_000;
 
 export async function runExternalCommandRunner(
 	runner: EvalRunnerV2,
@@ -133,8 +138,23 @@ export async function runExternalCommandRunner(
 	};
 }
 
+/**
+ * Run one eval subprocess to completion or to the timeout.
+ *
+ * A string runs through the shell, which is what an operator's
+ * `external-command` line and the verifier commands are. An argv array spawns
+ * the program directly, with no shell between the runner and the process
+ * whose exit code it reports. That distinction matters on the timeout path:
+ * `/bin/sh -c` on dash keeps the shell as the parent, so SIGTERM to the shell
+ * stops the shell alone while the program runs on to completion. A
+ * `clio-coder run` spawned that way finished its turn after the deadline,
+ * sealed a succeeded receipt with exit 0, and exited 0, and this runner
+ * reported the shell's signal death as exit 124 (issue #275). With an argv the
+ * signal reaches Clio itself, which seals the run as canceled with the exit
+ * status it then reports, so the receipt and the exit code agree either way.
+ */
 export function runShellCommand(
-	command: string,
+	command: string | readonly [string, ...string[]],
 	cwd: string,
 	timeoutMs: number,
 	env?: NodeJS.ProcessEnv,
@@ -153,14 +173,12 @@ export function runShellCommand(
 		const callLedgerFold = createEvalCallLedgerFold();
 		let timedOut = false;
 		let settled = false;
-		const child = spawn(command, {
-			cwd,
-			shell: true,
-			stdio: ["ignore", "pipe", "pipe"],
-			// The overlay is additive: an eval item pins where Clio writes its
-			// journal, and inherits everything else the operator's shell provides.
-			env: env === undefined ? process.env : { ...process.env, ...env },
-		});
+		// The overlay is additive: an eval item pins where Clio writes its
+		// journal, and inherits everything else the operator's shell provides.
+		const spawnEnv = env === undefined ? process.env : { ...process.env, ...env };
+		const [file, args, shell] = typeof command === "string" ? [command, [], true] : [command[0], command.slice(1), false];
+		const child = spawn(file, args, { cwd, shell, stdio: ["ignore", "pipe", "pipe"], env: spawnEnv });
+		const commandLine = typeof command === "string" ? command : command.join(" ");
 		child.stdout.setEncoding("utf8");
 		child.stderr.setEncoding("utf8");
 		child.stdout.on("data", (chunk: string) => {
@@ -177,14 +195,14 @@ export function runShellCommand(
 		const timer = setTimeout(() => {
 			timedOut = true;
 			child.kill("SIGTERM");
-			setTimeout(() => child.kill("SIGKILL"), 1000);
+			setTimeout(() => child.kill("SIGKILL"), TIMEOUT_KILL_GRACE_MS);
 		}, timeoutMs);
 		const finish = (exitCode: number): void => {
 			if (settled) return;
 			settled = true;
 			clearTimeout(timer);
 			resolve({
-				command,
+				command: commandLine,
 				exitCode,
 				stdout,
 				metricJsonl: metricCapture.finish(),

package/src/domains/eval/schema/suite.ts

@@ -13,6 +13,33 @@ export const CANONICAL_METRICS = [
 	// are absent in that case rather than zero, so a gate on cost can require
 	// measurement instead of reading silence as free.
 	"tokens.measured",
+	// Observed stdout terminal/retry facts, independent of task success. Opaque
+	// runs carry measured:false and no counters. Errored usage stays in inclusive
+	// tokens.* totals; errorTokens/errorCostUsd are known subtotals with coverage.
+	"provider.measured",
+	"provider.stopReason.stop",
+	"provider.stopReason.toolUse",
+	"provider.stopReason.length",
+	"provider.stopReason.error",
+	"provider.stopReason.aborted",
+	"provider.stopReason.other",
+	"provider.retryScheduled",
+	"provider.retryStarted",
+	"provider.retryCancelled",
+	"provider.retryExhausted",
+	"provider.retryRecovered",
+	"provider.errorUsageObservedCalls",
+	"provider.errorUsageUnobservedCalls",
+	"provider.errorUsageIncompleteCalls",
+	"provider.errorCostUnobservedCalls",
+	"provider.errorReasoningUnobservedCalls",
+	"provider.errorReasoningTokens",
+	"provider.errorTokens.input",
+	"provider.errorTokens.output",
+	"provider.errorTokens.total",
+	"provider.errorTokens.cacheRead",
+	"provider.errorTokens.cacheWrite",
+	"provider.errorCostUsd",
 	"latency.wallMs",
 	"latency.modelMs",
 	"tools.totalCalls",
@@ -22,6 +49,7 @@ export const CANONICAL_METRICS = [
 	"tools.read.outsideAllowed",
 	"tools.read.decoyHits",
 	"tools.calls.dispatch",
+	"tools.succeeded.dispatch",
 	"tools.blocked.bash",
 	"claims.unsupported",
 	"completion.reported",

package/src/domains/eval/schema/verdict.ts

@@ -39,7 +39,7 @@ export interface EvalTrackedMetricsV1 {
 	reasoningTokens: EvalSourcedNullableNumber;
 	toolCalls: EvalSourcedNumber;
 	toolErrors: EvalSourcedNumber;
-	ttftMsFirstCall: EvalSourcedNumber;
+	ttftMsFirstCall: EvalSourcedNullableNumber;
 	wallClockMs: EvalSourcedNumber;
 	contextTokensAtEnd: EvalSourcedNumber;
 	compactions: EvalSourcedNumber;
@@ -125,7 +125,7 @@ function parseTrackedMetrics(value: unknown, source: string): EvalTrackedMetrics
 		reasoningTokens: readSourcedNullableNumber(record.reasoningTokens, `${source}.reasoningTokens`),
 		toolCalls: readSourcedNumber(record.toolCalls, `${source}.toolCalls`),
 		toolErrors: readSourcedNumber(record.toolErrors, `${source}.toolErrors`),
-		ttftMsFirstCall: readSourcedNumber(record.ttftMsFirstCall, `${source}.ttftMsFirstCall`),
+		ttftMsFirstCall: readSourcedNullableNumber(record.ttftMsFirstCall, `${source}.ttftMsFirstCall`),
 		wallClockMs: readSourcedNumber(record.wallClockMs, `${source}.wallClockMs`),
 		contextTokensAtEnd: readSourcedNumber(record.contextTokensAtEnd, `${source}.contextTokensAtEnd`),
 		compactions: readSourcedNumber(record.compactions, `${source}.compactions`),

package/src/domains/eval/suites/resolve.ts

@@ -38,7 +38,19 @@ function resolveTargets(targets: ReadonlyArray<EvalSuiteTargetV2>, options: Reso
 	const filtered =
 		options.target === undefined ? [...targets] : targets.filter((target) => target.id === options.target);
 	const selected =
-		filtered.length > 0 ? filtered : options.target === undefined ? [...targets] : [{ id: options.target }];
+		filtered.length > 0
+			? filtered
+			: options.target === undefined
+				? [...targets]
+				: [
+						{
+							id: options.target,
+							// A one-row suite still owns its thinking dimension when the CLI
+							// substitutes the target/model pair. Dropping it silently changes
+							// an off baseline into the target's configured default.
+							...(targets.length === 1 && targets[0]?.thinking !== undefined ? { thinking: targets[0].thinking } : {}),
+						},
+					];
 	return selected.map((target) => ({
 		...target,
 		...(options.model === undefined ? {} : { model: options.model }),