@iowarp/clio-coder 0.3.7 → 0.3.9

package/src/domains/eval/provenance.ts

@@ -1,5 +1,9 @@
 import { spawnSync } from "node:child_process";
+import { readSettings } from "../../core/config.js";
 import { readClioVersion, resolvePackageRoot } from "../../core/package-root.js";
+import type { RunReceipt } from "../dispatch/types.js";
+import type { EvalServingConfigurationV1 } from "./schema/serving.js";
+import type { EvalSuiteTargetV2 } from "./schema/suite.js";
 import type { EvalClioProvenance, EvalEnvironmentProvenance } from "./types.js";
 
 export interface EvalProvenanceOptions {
@@ -7,6 +11,14 @@ export interface EvalProvenanceOptions {
 	commit?: string | null;
 }
 
+export interface EvalServingObservation {
+	targetId: string;
+	runtimeId: string | null;
+	modelId: string | null;
+	thinkingLevel: string | null;
+	compiledPromptHash: string | null;
+}
+
 export function evalClioProvenance(options: EvalProvenanceOptions = {}): EvalClioProvenance {
 	return {
 		version: readClioVersion(),
@@ -22,6 +34,45 @@ export function evalEnvironmentProvenance(): EvalEnvironmentProvenance {
 	};
 }
 
+/** Capture serving identity after the matrix has run while a local server is awake. */
+export async function evalServingConfiguration(
+	targets: ReadonlyArray<EvalSuiteTargetV2>,
+	observations: ReadonlyArray<EvalServingObservation>,
+): Promise<EvalServingConfigurationV1> {
+	const targetId = targets.length === 1 ? (targets[0]?.id ?? "unknown") : "multiple";
+	const configured = configuredTarget(targetId);
+	const target = targets.length === 1 ? targets[0] : undefined;
+	const runtimeId = consensus(observations.map((entry) => entry.runtimeId)) ?? configured?.runtime ?? null;
+	const modelId =
+		consensus(observations.map((entry) => entry.modelId)) ?? target?.model ?? configured?.defaultModel ?? null;
+	const props = configured?.url === undefined ? null : await readServingProps(configured.url, modelId);
+	return {
+		targetId,
+		runtimeId,
+		modelId,
+		serverBuild: props?.serverBuild ?? null,
+		total_slots: props?.totalSlots ?? null,
+		thinkingLevel: consensus(observations.map((entry) => entry.thinkingLevel)) ?? target?.thinking ?? null,
+		compiledPromptHash: consensus(observations.map((entry) => entry.compiledPromptHash)),
+	};
+}
+
+export function evalServingObservationFrom(
+	target: EvalSuiteTargetV2,
+	receipt: RunReceipt | null,
+	compiledPromptHashes: ReadonlyArray<string>,
+): EvalServingObservation {
+	const compiledPromptHash =
+		receipt?.staticCompositionHash ?? receipt?.compiledPromptHash ?? consensus(compiledPromptHashes);
+	return {
+		targetId: receipt?.targetId ?? target.id,
+		runtimeId: receipt?.runtimeId ?? null,
+		modelId: receipt?.wireModelId ?? target.model ?? null,
+		thinkingLevel: receipt?.runtimeResolution?.effectiveThinkingLevel ?? target.thinking ?? null,
+		compiledPromptHash,
+	};
+}
+
 function currentClioCommit(): string | null {
 	const result = spawnSync("git", ["rev-parse", "HEAD"], {
 		cwd: resolvePackageRoot(),
@@ -33,3 +84,69 @@ function currentClioCommit(): string | null {
 	const value = result.stdout.trim();
 	return value.length > 0 ? value : null;
 }
+
+interface ConfiguredServingTarget {
+	runtime: string;
+	url?: string;
+	defaultModel?: string;
+}
+
+function configuredTarget(targetId: string): ConfiguredServingTarget | null {
+	try {
+		const target = readSettings().targets.find((entry) => entry.id === targetId);
+		if (target === undefined) return null;
+		return {
+			runtime: target.runtime,
+			...(target.url === undefined ? {} : { url: target.url }),
+			...(target.defaultModel === undefined ? {} : { defaultModel: target.defaultModel }),
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingProps(
+	targetUrl: string,
+	modelId: string | null,
+): Promise<{ serverBuild: string | null; totalSlots: number | null } | null> {
+	const root = targetUrl.replace(/\/+$/u, "").replace(/\/v1$/u, "");
+	try {
+		const response = await fetch(`${root}/props`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		if (!isRecord(value)) return null;
+		const totalSlots = positiveInteger(value.total_slots) ?? (await readServingSlots(root, modelId));
+		return {
+			serverBuild: typeof value.build_info === "string" && value.build_info.length > 0 ? value.build_info : null,
+			totalSlots,
+		};
+	} catch {
+		return null;
+	}
+}
+
+async function readServingSlots(root: string, modelId: string | null): Promise<number | null> {
+	if (modelId === null) return null;
+	try {
+		const query = new URLSearchParams({ model: modelId });
+		const response = await fetch(`${root}/slots?${query}`, { signal: AbortSignal.timeout(5_000) });
+		if (!response.ok) return null;
+		const value: unknown = await response.json();
+		return Array.isArray(value) && value.length > 0 ? value.length : null;
+	} catch {
+		return null;
+	}
+}
+
+function consensus(values: ReadonlyArray<string | null>): string | null {
+	const known = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))];
+	return known.length === 1 ? (known[0] ?? null) : null;
+}
+
+function positiveInteger(value: unknown): number | null {
+	return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/domains/eval/reports/comparison.ts

@@ -0,0 +1,128 @@
+import type { EvalCompareV4Summary } from "../compare/compare.js";
+import { renderEvalComparisonV4 } from "../compare/compare.js";
+
+export type EvalComparisonReportFormat = "text" | "json" | "md" | "junit";
+
+export function renderEvalComparisonReportV1(
+	summary: EvalCompareV4Summary,
+	format: EvalComparisonReportFormat,
+): string {
+	if (format === "json") return `${JSON.stringify(summary, null, 2)}\n`;
+	if (format === "md") return renderMarkdown(summary);
+	if (format === "junit") return renderJunit(summary);
+	return renderEvalComparisonV4(summary);
+}
+
+function renderMarkdown(summary: EvalCompareV4Summary): string {
+	const rows = summary.behavioralMetrics.map(
+		(row) =>
+			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(`${row.target.id}/${row.target.model ?? "none"}`)} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
+	);
+	const scenarioRows = summary.scenarioReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	const roleRows = summary.roleReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	return [
+		`# Eval comparison ${summary.baselineEvalId} → ${summary.candidateEvalId}`,
+		"",
+		`Behavioral hard gate: **${summary.hardGate.pass ? "pass" : "fail"}**`,
+		...summary.hardGate.failures.map(
+			(failure) =>
+				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${failure.target.id}/${failure.target.model ?? "none"} / ${failure.metric}: ${failure.change}`,
+		),
+		...summary.envelopeMismatches.map(
+			(mismatch) =>
+				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${mismatch.target.id}/${mismatch.target.model ?? "none"}: ${mismatch.fields.join(", ")}`,
+		),
+		...summary.affectedCorpusResults.map(
+			(result) => `- Affected corpus result: ${result.scenarioId} / ${result.role}: ${result.changedFields.join(", ")}`,
+		),
+		`Pass-rate delta: ${(summary.passRateDelta * 100).toFixed(2)}%`,
+		`Token delta: ${summary.tokenDelta === null ? "unmeasured" : summary.tokenDelta}`,
+		`Wall-time delta ms: ${summary.wallTimeDelta}`,
+		"",
+		"| Scenario | Role | Target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Change | Variance | Comparability | Gate |",
+		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|",
+		...rows,
+		"",
+		"## Per-scenario baseline/candidate report",
+		"",
+		"| Scenario | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...scenarioRows,
+		"",
+		"## Per-role baseline/candidate report",
+		"",
+		"| Role | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...roleRows,
+		"",
+	].join("\n");
+}
+
+function renderJunit(summary: EvalCompareV4Summary): string {
+	const failures = new Set(
+		summary.hardGate.failures.map((failure) =>
+			JSON.stringify([failure.scenarioId, failure.role, failure.target.id, failure.target.model, failure.metric]),
+		),
+	);
+	const represented = new Set<string>();
+	const cases = summary.behavioralMetrics.map((row) => {
+		const name = `${row.scenarioId}[${row.role}:${row.target.id}:${row.target.model ?? "none"}].${row.metric}`;
+		const key = JSON.stringify([row.scenarioId, row.role, row.target.id, row.target.model, row.metric]);
+		represented.add(key);
+		const detail = `change=${row.change} variance=${row.varianceChange} baseline=${format(row.baseline.mean)} candidate=${format(row.candidate.mean)}`;
+		return failures.has(key)
+			? `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><failure message="${escapeXml(row.change)}">${escapeXml(detail)}</failure></testcase>`
+			: `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><system-out>${escapeXml(detail)}</system-out></testcase>`;
+	});
+	for (const failure of summary.hardGate.failures) {
+		const key = JSON.stringify([
+			failure.scenarioId,
+			failure.role,
+			failure.target.id,
+			failure.target.model,
+			failure.metric,
+		]);
+		if (represented.has(key)) continue;
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].${failure.metric}`;
+		cases.push(
+			`  <testcase classname="eval.behavior.hard" name="${escapeXml(name)}"><failure message="${escapeXml(failure.change)}">hard behavioral gate</failure></testcase>`,
+		);
+	}
+	for (const failure of summary.hardGate.envelopeFailures) {
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].execution-envelope`;
+		cases.push(
+			`  <testcase classname="eval.behavior.envelope" name="${escapeXml(name)}"><failure message="incomparable">${escapeXml(failure.fields.join(", "))}</failure></testcase>`,
+		);
+	}
+	return [
+		`<testsuite name="eval-comparison" tests="${cases.length}" failures="${summary.hardGate.failures.length + summary.hardGate.envelopeFailures.length}">`,
+		...cases,
+		"</testsuite>",
+		"",
+	].join("\n");
+}
+
+function changeCounts(counts: {
+	improved: number;
+	regressed: number;
+	unchanged: number;
+	incomparable: number;
+}): string {
+	return `improved ${counts.improved}, regressed ${counts.regressed}, unchanged ${counts.unchanged}, incomparable ${counts.incomparable}`;
+}
+
+function format(value: number | null): string {
+	return value === null ? "null" : Number.isInteger(value) ? String(value) : value.toFixed(4);
+}
+
+function cell(value: string): string {
+	return value.replaceAll("|", "\\|");
+}
+
+function escapeXml(value: string): string {
+	return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
+}

package/src/domains/eval/reports/junit.ts

@@ -1,17 +1,31 @@
 import type { EvalArtifactV4 } from "../schema/artifact.js";
 
 export function renderEvalJunitReportV4(artifact: EvalArtifactV4): string {
+	let failures = 0;
+	let skipped = 0;
 	const cases = artifact.results
 		.map((result) => {
 			const name = escapeXml(
 				`${result.taskId}[${result.target.id}:${result.target.model ?? "default"}:${result.repeatIndex}]`,
 			);
-			if (result.pass) return `  <testcase name="${name}" />`;
-			return `  <testcase name="${name}"><failure message="${escapeXml(result.failureClass ?? "failed")}" /></testcase>`;
+			if (!result.pass) {
+				failures += 1;
+				return `  <testcase name="${name}"><failure message="${escapeXml(result.failureClass ?? "failed")}" /></testcase>`;
+			}
+			const outcome = result.behavioral?.outcome;
+			if (outcome === "behavioral_failure" || outcome === "infrastructure_failure") {
+				failures += 1;
+				return `  <testcase name="${name}"><failure message="${escapeXml(outcome)}" /></testcase>`;
+			}
+			if (outcome === "unknown" || outcome === "unmeasured") {
+				skipped += 1;
+				return `  <testcase name="${name}"><skipped message="behavioral ${escapeXml(outcome)}" /></testcase>`;
+			}
+			return `  <testcase name="${name}" />`;
 		})
 		.join("\n");
 	return [
-		`<testsuite name="${escapeXml(artifact.suite.id)}" tests="${artifact.summary.runs}" failures="${artifact.summary.failed}">`,
+		`<testsuite name="${escapeXml(artifact.suite.id)}" tests="${artifact.summary.runs}" failures="${failures}" skipped="${skipped}">`,
 		cases,
 		"</testsuite>",
 		"",

package/src/domains/eval/reports/markdown.ts

@@ -8,11 +8,11 @@ export function renderEvalMarkdownReportV4(artifact: EvalArtifactV4): string {
 		`Target: ${artifact.matrix.target}`,
 		`Pass rate: ${(artifact.summary.passRate * 100).toFixed(2)}%`,
 		"",
-		"| Task | Target | Model | Repeat | Pass | Failure |",
-		"|---|---|---|---:|---|---|",
+		"| Task | Role | Target | Model | Repeat | Result | Behavioral | Failure |",
+		"|---|---|---|---|---:|---|---|---|",
 		...artifact.results.map(
 			(result) =>
-				`| ${result.taskId} | ${result.target.id} | ${result.target.model ?? ""} | ${result.repeatIndex} | ${result.pass ? "pass" : "fail"} | ${result.failureClass ?? ""} |`,
+				`| ${result.taskId} | ${result.behavioralMetrics?.role ?? ""} | ${result.target.id} | ${result.target.model ?? ""} | ${result.repeatIndex} | ${result.pass ? "pass" : "fail"} | ${result.behavioral?.outcome ?? "unmeasured"} | ${result.failureClass ?? ""} |`,
 		),
 		"",
 	];

package/src/domains/eval/reports/text.ts

@@ -2,6 +2,15 @@ import type { EvalArtifactV4 } from "../schema/artifact.js";
 
 export function renderEvalTextReportV4(artifact: EvalArtifactV4): string {
 	const tokens = artifact.summary.tokens;
+	const behavioral = artifact.results.flatMap((result) =>
+		result.behavioral === undefined ? [] : [result.behavioral.outcome],
+	);
+	const behavioralSummary =
+		behavioral.length === 0
+			? []
+			: [
+					`behavioral: pass=${count(behavioral, "pass")} failure=${count(behavioral, "behavioral_failure")} unknown=${count(behavioral, "unknown")} unmeasured=${count(behavioral, "unmeasured")} infrastructure=${count(behavioral, "infrastructure_failure")}`,
+				];
 	return [
 		`eval: ${artifact.evalId}`,
 		`suite: ${artifact.suite.id}`,
@@ -20,6 +29,11 @@ export function renderEvalTextReportV4(artifact: EvalArtifactV4): string {
 				? `tokens total: ${tokens.total}`
 				: `tokens total: ${tokens.total} (measured in ${tokens.measuredRuns} of ${tokens.runs} runs)`,
 		`wall time ms: ${artifact.summary.wallTimeMs}`,
+		...behavioralSummary,
 		"",
 	].join("\n");
 }
+
+function count(values: ReadonlyArray<string>, wanted: string): number {
+	return values.filter((value) => value === wanted).length;
+}

package/src/domains/eval/run-compare.ts

@@ -1,8 +1,28 @@
 import { subtractEvalHarnessMetrics } from "./harness-metrics.js";
+import type { EvalMetricSource } from "./schema/verdict.js";
 import type { EvalFailureClass, EvalHarnessMetrics, EvalRunArtifact, EvalRunRecord, EvalSummary } from "./types.js";
 
 export const EVAL_COMPARE_MATCHING_RULE = "taskId+repeatIndex";
 
+export class EvalTrackedMetricSourceMismatchError extends Error {
+	constructor(metric: string, baseline: ReadonlyArray<EvalMetricSource>, candidate: ReadonlyArray<EvalMetricSource>) {
+		super(
+			`tracked metric ${metric} cannot compare estimated and measured values (baseline=${baseline.join(",") || "none"}, candidate=${candidate.join(",") || "none"})`,
+		);
+		this.name = "EvalTrackedMetricSourceMismatchError";
+	}
+}
+
+/** Refuse a delta when exactly one side contains estimated observations. */
+export function assertComparableTrackedMetricSources(
+	metric: string,
+	baseline: ReadonlyArray<EvalMetricSource>,
+	candidate: ReadonlyArray<EvalMetricSource>,
+): void {
+	if (baseline.includes("estimated") === candidate.includes("estimated")) return;
+	throw new EvalTrackedMetricSourceMismatchError(metric, baseline, candidate);
+}
+
 export interface EvalCompareTotals {
 	passed: number;
 	failed: number;

package/src/domains/eval/runners/clio-run.ts

@@ -1,5 +1,11 @@
+import { isAbsolute, relative, resolve, sep } from "node:path";
 import { shellQuote } from "../../../core/shell-quote.js";
-import { evidenceMetricsFromReceipt, receiptFromRunJsonStdout } from "../metrics/evidence.js";
+import { clioStateDir } from "../../../core/xdg.js";
+import {
+	evidenceMetricsFromReceipt,
+	readRunEnvelopeForReceipt,
+	receiptFromRunJsonStdout,
+} from "../metrics/evidence.js";
 import { streamInvariantMetrics } from "../metrics/invariants.js";
 import { tokenMetricEntries } from "../metrics/token-stream.js";
 import type { EvalRunnerV2, EvalSuiteTargetV2 } from "../schema/suite.js";
@@ -12,6 +18,7 @@ export async function runClioRunRunner(
 	timeoutMs: number,
 	target: EvalSuiteTargetV2,
 	env?: NodeJS.ProcessEnv,
+	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
 ): Promise<EvalRunnerOutput> {
 	const prompt = runner.prompt ?? "";
 	const args = [
@@ -23,6 +30,7 @@ export async function runClioRunRunner(
 		shellQuote(target.id),
 		...(target.model === undefined ? [] : ["--model", shellQuote(target.model)]),
 		...(target.thinking === undefined ? [] : ["--thinking", shellQuote(target.thinking)]),
+		...(runner.autonomy === undefined ? [] : ["--autonomy", runner.autonomy]),
 		shellQuote(prompt),
 	];
 	const result = await runShellCommand(`${process.execPath} ${args.join(" ")}`, cwd, runner.timeoutMs ?? timeoutMs, env);
@@ -31,10 +39,16 @@ export async function runClioRunRunner(
 	const tokens = result.usage;
 	const toolMetricStream = result.metricJsonl.length > 0 ? result.metricJsonl : result.stdout;
 	const tools = toolCallMetricsFromJsonl(toolMetricStream);
+	const behavioralTools = toolBehaviorMetricEntriesFromJsonl(toolMetricStream, cwd, readObservation);
 	// Evidence metrics resolve only from the sealed receipt the --agent path
 	// prints; a runner without a receipt leaves them absent so any gate on
 	// them fails closed instead of reading prose labels.
 	const receipt = receiptFromRunJsonStdout(result.stdout);
+	// The canonical trust metrics authenticate the receipt against the ledger
+	// row the child wrote; a stdout receipt on its own only ever reads as an
+	// unchecked seal.
+	const envelope =
+		receipt === null ? null : readRunEnvelopeForReceipt(receipt, env?.CLIO_CODER_STATE_DIR ?? clioStateDir());
 	return {
 		assignmentId: receipt === null ? null : (receipt.lineage?.rootRunId ?? receipt.runId),
 		terminalReceiptDigest: receipt?.integrity.digest ?? null,
@@ -51,8 +65,9 @@ export async function runClioRunRunner(
 			"tools.totalCalls": tools.totalCalls,
 			"tools.failed": tools.failed,
 			"tools.blocked": tools.blocked,
+			...behavioralTools,
 			"verifier.exitCode": result.exitCode,
-			...(receipt === null ? {} : evidenceMetricsFromReceipt(receipt)),
+			...(receipt === null ? {} : evidenceMetricsFromReceipt(receipt, { envelope })),
 			...(receipt === null
 				? {}
 				: { "evidence.qualityLabel": receipt.quality.typedValidations.length > 0 ? "measured" : "unmeasured" }),
@@ -60,8 +75,11 @@ export async function runClioRunRunner(
 		artifacts: {
 			stdout: result.stdout,
 			stderr: result.stderr,
+			callLedger: JSON.stringify(result.ledgerEntries),
 			...(receipt === null ? {} : { receipt: JSON.stringify(receipt) }),
 		},
+		receipt,
+		ledgerEntries: result.ledgerEntries,
 	};
 }
 
@@ -118,6 +136,125 @@ export function toolCallMetricsFromJsonl(stdout: string): ToolCallMetrics {
 	return canonicalFinishes.totalCalls > 0 ? canonicalFinishes : executionEnds;
 }
 
+interface BehavioralToolTerminal {
+	callId: string | null;
+	tool: string;
+	outcome: ToolOutcome;
+}
+
+/**
+ * Reduce the same terminal stream to bounded behavioral facts. Tool names are
+ * a dynamic metric suffix because extensions may add tools. Read paths become
+ * bounded counters against the suite's explicit public allowlist and decoy
+ * list; they never become behavioral fact values or evidence excerpts.
+ */
+export function toolBehaviorMetricEntriesFromJsonl(
+	stdout: string,
+	cwd: string,
+	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
+): Record<string, number> {
+	const starts = new Map<string, { tool: string; path: string | null }>();
+	const readPaths = new Set<string>();
+	const executionEnds: BehavioralToolTerminal[] = [];
+	const canonicalFinishes: BehavioralToolTerminal[] = [];
+	const seenExecution = new Set<string>();
+	const seenCanonical = new Set<string>();
+
+	for (const line of stdout.split(/\r?\n/)) {
+		if (line.trim().length === 0) continue;
+		let event: Record<string, unknown>;
+		try {
+			const parsed: unknown = JSON.parse(line);
+			if (!isRecord(parsed)) continue;
+			event = parsed;
+		} catch {
+			continue;
+		}
+		if (event.type === "tool_execution_start") {
+			const callId = stringField(event, "toolCallId");
+			const tool = stringField(event, "toolName");
+			if (callId === undefined || tool === undefined) continue;
+			const path = tool === "read" && isRecord(event.args) ? toolPath(event.args) : null;
+			starts.set(callId, { tool, path });
+			if (path !== null) readPaths.add(normalizeObservedPath(cwd, path));
+			continue;
+		}
+		if (event.type === "tool_execution_end") {
+			const callId = stringField(event, "toolCallId") ?? null;
+			if (callId !== null && seenExecution.has(callId)) continue;
+			if (callId !== null) seenExecution.add(callId);
+			const tool = stringField(event, "toolName") ?? (callId === null ? undefined : starts.get(callId)?.tool);
+			if (tool === undefined) continue;
+			executionEnds.push({ callId, tool, outcome: toolOutcome(event) ?? (event.isError === true ? "error" : "ok") });
+			continue;
+		}
+		if (event.type !== "clio_tool_finish" || !isRecord(event.payload)) continue;
+		const outcome = toolOutcome(event.payload);
+		const tool = stringField(event.payload, "tool");
+		if (outcome === undefined || tool === undefined) continue;
+		const callId = stringField(event.payload, "toolCallId") ?? stringField(event, "toolCallId") ?? null;
+		if (callId !== null && seenCanonical.has(callId)) continue;
+		if (callId !== null) seenCanonical.add(callId);
+		canonicalFinishes.push({ callId, tool, outcome });
+	}
+
+	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
+	const calls = new Map<string, number>();
+	const blocked = new Map<string, number>();
+	for (const terminal of terminals) {
+		const tool = metricToolName(terminal.tool);
+		calls.set(tool, (calls.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
+	}
+	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...blocked.keys()]);
+	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
+	for (const tool of [...namedTools].sort()) {
+		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
+		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
+	}
+	if (readObservation !== undefined) {
+		const allowed = readObservation.allowedPaths.map((path) => normalizeObservedPath(cwd, path));
+		const decoys = readObservation.decoyPaths.map((path) => normalizeObservedPath(cwd, path));
+		entries["tools.read.outsideAllowed"] = [...readPaths].filter(
+			(path) => !allowed.some((root) => pathWithin(path, root)),
+		).length;
+		entries["tools.read.decoyHits"] = [...readPaths].filter((path) =>
+			decoys.some((root) => pathWithin(path, root)),
+		).length;
+	}
+	return entries;
+}
+
+function toolPath(args: Record<string, unknown>): string | null {
+	for (const field of ["path", "filePath", "file_path"]) {
+		const value = args[field];
+		if (typeof value === "string" && value.length > 0 && value.length <= 4_096) return value;
+	}
+	return null;
+}
+
+function normalizeObservedPath(cwd: string, path: string): string {
+	const absolute = resolve(cwd, path);
+	const local = relative(cwd, absolute);
+	return (isAbsolute(path) && (local.startsWith("..") || isAbsolute(local)) ? absolute : local || ".")
+		.split(sep)
+		.join("/");
+}
+
+function pathWithin(path: string, root: string): boolean {
+	if (root === ".") return !isAbsolute(path) && path !== ".." && !path.startsWith("../");
+	return path === root || path.startsWith(`${root}/`);
+}
+
+function metricToolName(tool: string): string {
+	return (
+		tool
+			.toLowerCase()
+			.replaceAll(/[^a-z0-9_-]/gu, "_")
+			.slice(0, 64) || "unknown"
+	);
+}
+
 function recordToolOutcome(metrics: ToolCallMetrics, outcome: ToolOutcome): void {
 	metrics.totalCalls += 1;
 	if (outcome === "error") metrics.failed += 1;

package/src/domains/eval/runners/external-command.ts

@@ -1,5 +1,8 @@
 import { spawn } from "node:child_process";
 import { performance } from "node:perf_hooks";
+import type { RunReceipt } from "../../dispatch/types.js";
+import type { SessionEntry } from "../../session/entries.js";
+import { createEvalCallLedgerFold } from "../metrics/call-ledger-stream.js";
 import {
 	addFleetLoopObservations,
 	createFleetLoopFold,
@@ -33,6 +36,10 @@ export interface EvalRunnerOutput {
 	wallTimeMs: number;
 	metrics: Record<string, number | string | boolean | null>;
 	artifacts: Record<string, string | string[] | null>;
+	/** Parsed sealed receipt when this runner exposes one. */
+	receipt?: RunReceipt | null;
+	/** Structured per-call facts folded before bounded stdout truncation. */
+	ledgerEntries?: SessionEntry[];
 }
 
 export interface ShellCommandResult {
@@ -46,6 +53,7 @@ export interface ShellCommandResult {
 	/** Wire-stream structural invariants folded live, for the same reason. */
 	streamInvariants: EvalStreamInvariants;
 	fleetLoops: EvalFleetLoopObservation;
+	ledgerEntries: SessionEntry[];
 	stderr: string;
 	wallTimeMs: number;
 	timedOut: boolean;
@@ -75,6 +83,7 @@ export async function runExternalCommandRunner(
 	let usage = UNMEASURED_TOKEN_USAGE;
 	let streamInvariants = EMPTY_STREAM_INVARIANTS;
 	let fleetLoops = EMPTY_FLEET_LOOP_OBSERVATION;
+	const ledgerEntries: SessionEntry[] = [];
 	for (const command of commands) {
 		const result = await runShellCommand(command, cwd, runner.timeoutMs ?? timeoutMs, env);
 		stdout = appendLimited(stdout, result.stdout);
@@ -83,6 +92,7 @@ export async function runExternalCommandRunner(
 		usage = addTokenStreamUsage(usage, result.usage);
 		streamInvariants = addStreamInvariants(streamInvariants, result.streamInvariants);
 		fleetLoops = addFleetLoopObservations(fleetLoops, result.fleetLoops);
+		ledgerEntries.push(...result.ledgerEntries);
 		if (result.exitCode !== 0) {
 			return {
 				assignmentId: null,
@@ -99,6 +109,7 @@ export async function runExternalCommandRunner(
 					"verifier.exitCode": result.exitCode,
 				},
 				artifacts: {},
+				ledgerEntries,
 			};
 		}
 	}
@@ -117,6 +128,7 @@ export async function runExternalCommandRunner(
 			"verifier.exitCode": 0,
 		},
 		artifacts: {},
+		ledgerEntries,
 	};
 }
 
@@ -137,6 +149,7 @@ export function runShellCommand(
 		const usageFold = createTokenUsageFold();
 		const streamFold = createStreamInvariantFold();
 		const fleetLoopFold = createFleetLoopFold();
+		const callLedgerFold = createEvalCallLedgerFold();
 		let timedOut = false;
 		let settled = false;
 		const child = spawn(command, {
@@ -155,6 +168,7 @@ export function runShellCommand(
 			usageFold.push(chunk);
 			streamFold.push(chunk);
 			fleetLoopFold.push(chunk);
+			callLedgerFold.push(chunk);
 		});
 		child.stderr.on("data", (chunk: string) => {
 			stderr = appendLimited(stderr, chunk);
@@ -176,6 +190,7 @@ export function runShellCommand(
 				usage: usageFold.usage(),
 				streamInvariants: streamFold.invariants(),
 				fleetLoops: fleetLoopFold.observation(),
+				ledgerEntries: callLedgerFold.entries(),
 				stderr,
 				wallTimeMs: Math.round(performance.now() - started),
 				timedOut,
@@ -284,9 +299,11 @@ function compactMetricEvent(event: Record<string, unknown>): Record<string, unkn
 			toolName,
 			...(toolName === "dispatch" && isRecord(event.args)
 				? { args: event.args }
-				: toolName === "code_nav" && isRecord(event.args)
-					? { args: { mode: event.args.mode } }
-					: {}),
+				: toolName === "read" && isRecord(event.args)
+					? { args: boundedReadArgs(event.args) }
+					: toolName === "code_nav" && isRecord(event.args)
+						? { args: { mode: event.args.mode } }
+						: {}),
 		};
 	}
 	if (type === "tool_execution_end") {
@@ -313,6 +330,14 @@ function compactMetricEvent(event: Record<string, unknown>): Record<string, unkn
 	};
 }
 
+function boundedReadArgs(args: Record<string, unknown>): Record<string, string> {
+	for (const field of ["path", "filePath", "file_path"]) {
+		const value = args[field];
+		if (typeof value === "string" && value.length > 0) return { [field]: value.slice(0, 4_096) };
+	}
+	return {};
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }