@iowarp/clio-coder 0.3.8 → 0.3.9

package/src/domains/eval/reports/comparison.ts

@@ -0,0 +1,128 @@
+import type { EvalCompareV4Summary } from "../compare/compare.js";
+import { renderEvalComparisonV4 } from "../compare/compare.js";
+
+export type EvalComparisonReportFormat = "text" | "json" | "md" | "junit";
+
+export function renderEvalComparisonReportV1(
+	summary: EvalCompareV4Summary,
+	format: EvalComparisonReportFormat,
+): string {
+	if (format === "json") return `${JSON.stringify(summary, null, 2)}\n`;
+	if (format === "md") return renderMarkdown(summary);
+	if (format === "junit") return renderJunit(summary);
+	return renderEvalComparisonV4(summary);
+}
+
+function renderMarkdown(summary: EvalCompareV4Summary): string {
+	const rows = summary.behavioralMetrics.map(
+		(row) =>
+			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(`${row.target.id}/${row.target.model ?? "none"}`)} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
+	);
+	const scenarioRows = summary.scenarioReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	const roleRows = summary.roleReports.map(
+		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
+	);
+	return [
+		`# Eval comparison ${summary.baselineEvalId} → ${summary.candidateEvalId}`,
+		"",
+		`Behavioral hard gate: **${summary.hardGate.pass ? "pass" : "fail"}**`,
+		...summary.hardGate.failures.map(
+			(failure) =>
+				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${failure.target.id}/${failure.target.model ?? "none"} / ${failure.metric}: ${failure.change}`,
+		),
+		...summary.envelopeMismatches.map(
+			(mismatch) =>
+				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${mismatch.target.id}/${mismatch.target.model ?? "none"}: ${mismatch.fields.join(", ")}`,
+		),
+		...summary.affectedCorpusResults.map(
+			(result) => `- Affected corpus result: ${result.scenarioId} / ${result.role}: ${result.changedFields.join(", ")}`,
+		),
+		`Pass-rate delta: ${(summary.passRateDelta * 100).toFixed(2)}%`,
+		`Token delta: ${summary.tokenDelta === null ? "unmeasured" : summary.tokenDelta}`,
+		`Wall-time delta ms: ${summary.wallTimeDelta}`,
+		"",
+		"| Scenario | Role | Target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Change | Variance | Comparability | Gate |",
+		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|",
+		...rows,
+		"",
+		"## Per-scenario baseline/candidate report",
+		"",
+		"| Scenario | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...scenarioRows,
+		"",
+		"## Per-role baseline/candidate report",
+		"",
+		"| Role | Metric changes | Variance changes |",
+		"|---|---|---|",
+		...roleRows,
+		"",
+	].join("\n");
+}
+
+function renderJunit(summary: EvalCompareV4Summary): string {
+	const failures = new Set(
+		summary.hardGate.failures.map((failure) =>
+			JSON.stringify([failure.scenarioId, failure.role, failure.target.id, failure.target.model, failure.metric]),
+		),
+	);
+	const represented = new Set<string>();
+	const cases = summary.behavioralMetrics.map((row) => {
+		const name = `${row.scenarioId}[${row.role}:${row.target.id}:${row.target.model ?? "none"}].${row.metric}`;
+		const key = JSON.stringify([row.scenarioId, row.role, row.target.id, row.target.model, row.metric]);
+		represented.add(key);
+		const detail = `change=${row.change} variance=${row.varianceChange} baseline=${format(row.baseline.mean)} candidate=${format(row.candidate.mean)}`;
+		return failures.has(key)
+			? `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><failure message="${escapeXml(row.change)}">${escapeXml(detail)}</failure></testcase>`
+			: `  <testcase classname="eval.behavior.${escapeXml(row.family)}" name="${escapeXml(name)}"><system-out>${escapeXml(detail)}</system-out></testcase>`;
+	});
+	for (const failure of summary.hardGate.failures) {
+		const key = JSON.stringify([
+			failure.scenarioId,
+			failure.role,
+			failure.target.id,
+			failure.target.model,
+			failure.metric,
+		]);
+		if (represented.has(key)) continue;
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].${failure.metric}`;
+		cases.push(
+			`  <testcase classname="eval.behavior.hard" name="${escapeXml(name)}"><failure message="${escapeXml(failure.change)}">hard behavioral gate</failure></testcase>`,
+		);
+	}
+	for (const failure of summary.hardGate.envelopeFailures) {
+		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].execution-envelope`;
+		cases.push(
+			`  <testcase classname="eval.behavior.envelope" name="${escapeXml(name)}"><failure message="incomparable">${escapeXml(failure.fields.join(", "))}</failure></testcase>`,
+		);
+	}
+	return [
+		`<testsuite name="eval-comparison" tests="${cases.length}" failures="${summary.hardGate.failures.length + summary.hardGate.envelopeFailures.length}">`,
+		...cases,
+		"</testsuite>",
+		"",
+	].join("\n");
+}
+
+function changeCounts(counts: {
+	improved: number;
+	regressed: number;
+	unchanged: number;
+	incomparable: number;
+}): string {
+	return `improved ${counts.improved}, regressed ${counts.regressed}, unchanged ${counts.unchanged}, incomparable ${counts.incomparable}`;
+}
+
+function format(value: number | null): string {
+	return value === null ? "null" : Number.isInteger(value) ? String(value) : value.toFixed(4);
+}
+
+function cell(value: string): string {
+	return value.replaceAll("|", "\\|");
+}
+
+function escapeXml(value: string): string {
+	return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
+}

package/src/domains/eval/reports/junit.ts

@@ -1,17 +1,31 @@
 import type { EvalArtifactV4 } from "../schema/artifact.js";
 
 export function renderEvalJunitReportV4(artifact: EvalArtifactV4): string {
+	let failures = 0;
+	let skipped = 0;
 	const cases = artifact.results
 		.map((result) => {
 			const name = escapeXml(
 				`${result.taskId}[${result.target.id}:${result.target.model ?? "default"}:${result.repeatIndex}]`,
 			);
-			if (result.pass) return `  <testcase name="${name}" />`;
-			return `  <testcase name="${name}"><failure message="${escapeXml(result.failureClass ?? "failed")}" /></testcase>`;
+			if (!result.pass) {
+				failures += 1;
+				return `  <testcase name="${name}"><failure message="${escapeXml(result.failureClass ?? "failed")}" /></testcase>`;
+			}
+			const outcome = result.behavioral?.outcome;
+			if (outcome === "behavioral_failure" || outcome === "infrastructure_failure") {
+				failures += 1;
+				return `  <testcase name="${name}"><failure message="${escapeXml(outcome)}" /></testcase>`;
+			}
+			if (outcome === "unknown" || outcome === "unmeasured") {
+				skipped += 1;
+				return `  <testcase name="${name}"><skipped message="behavioral ${escapeXml(outcome)}" /></testcase>`;
+			}
+			return `  <testcase name="${name}" />`;
 		})
 		.join("\n");
 	return [
-		`<testsuite name="${escapeXml(artifact.suite.id)}" tests="${artifact.summary.runs}" failures="${artifact.summary.failed}">`,
+		`<testsuite name="${escapeXml(artifact.suite.id)}" tests="${artifact.summary.runs}" failures="${failures}" skipped="${skipped}">`,
 		cases,
 		"</testsuite>",
 		"",

package/src/domains/eval/reports/markdown.ts

@@ -8,11 +8,11 @@ export function renderEvalMarkdownReportV4(artifact: EvalArtifactV4): string {
 		`Target: ${artifact.matrix.target}`,
 		`Pass rate: ${(artifact.summary.passRate * 100).toFixed(2)}%`,
 		"",
-		"| Task | Target | Model | Repeat | Pass | Failure |",
-		"|---|---|---|---:|---|---|",
+		"| Task | Role | Target | Model | Repeat | Result | Behavioral | Failure |",
+		"|---|---|---|---|---:|---|---|---|",
 		...artifact.results.map(
 			(result) =>
-				`| ${result.taskId} | ${result.target.id} | ${result.target.model ?? ""} | ${result.repeatIndex} | ${result.pass ? "pass" : "fail"} | ${result.failureClass ?? ""} |`,
+				`| ${result.taskId} | ${result.behavioralMetrics?.role ?? ""} | ${result.target.id} | ${result.target.model ?? ""} | ${result.repeatIndex} | ${result.pass ? "pass" : "fail"} | ${result.behavioral?.outcome ?? "unmeasured"} | ${result.failureClass ?? ""} |`,
 		),
 		"",
 	];

package/src/domains/eval/reports/text.ts

@@ -2,6 +2,15 @@ import type { EvalArtifactV4 } from "../schema/artifact.js";
 
 export function renderEvalTextReportV4(artifact: EvalArtifactV4): string {
 	const tokens = artifact.summary.tokens;
+	const behavioral = artifact.results.flatMap((result) =>
+		result.behavioral === undefined ? [] : [result.behavioral.outcome],
+	);
+	const behavioralSummary =
+		behavioral.length === 0
+			? []
+			: [
+					`behavioral: pass=${count(behavioral, "pass")} failure=${count(behavioral, "behavioral_failure")} unknown=${count(behavioral, "unknown")} unmeasured=${count(behavioral, "unmeasured")} infrastructure=${count(behavioral, "infrastructure_failure")}`,
+				];
 	return [
 		`eval: ${artifact.evalId}`,
 		`suite: ${artifact.suite.id}`,
@@ -20,6 +29,11 @@ export function renderEvalTextReportV4(artifact: EvalArtifactV4): string {
 				? `tokens total: ${tokens.total}`
 				: `tokens total: ${tokens.total} (measured in ${tokens.measuredRuns} of ${tokens.runs} runs)`,
 		`wall time ms: ${artifact.summary.wallTimeMs}`,
+		...behavioralSummary,
 		"",
 	].join("\n");
 }
+
+function count(values: ReadonlyArray<string>, wanted: string): number {
+	return values.filter((value) => value === wanted).length;
+}

package/src/domains/eval/run-compare.ts

@@ -1,8 +1,28 @@
 import { subtractEvalHarnessMetrics } from "./harness-metrics.js";
+import type { EvalMetricSource } from "./schema/verdict.js";
 import type { EvalFailureClass, EvalHarnessMetrics, EvalRunArtifact, EvalRunRecord, EvalSummary } from "./types.js";
 
 export const EVAL_COMPARE_MATCHING_RULE = "taskId+repeatIndex";
 
+export class EvalTrackedMetricSourceMismatchError extends Error {
+	constructor(metric: string, baseline: ReadonlyArray<EvalMetricSource>, candidate: ReadonlyArray<EvalMetricSource>) {
+		super(
+			`tracked metric ${metric} cannot compare estimated and measured values (baseline=${baseline.join(",") || "none"}, candidate=${candidate.join(",") || "none"})`,
+		);
+		this.name = "EvalTrackedMetricSourceMismatchError";
+	}
+}
+
+/** Refuse a delta when exactly one side contains estimated observations. */
+export function assertComparableTrackedMetricSources(
+	metric: string,
+	baseline: ReadonlyArray<EvalMetricSource>,
+	candidate: ReadonlyArray<EvalMetricSource>,
+): void {
+	if (baseline.includes("estimated") === candidate.includes("estimated")) return;
+	throw new EvalTrackedMetricSourceMismatchError(metric, baseline, candidate);
+}
+
 export interface EvalCompareTotals {
 	passed: number;
 	failed: number;

package/src/domains/eval/runners/clio-run.ts

@@ -1,3 +1,4 @@
+import { isAbsolute, relative, resolve, sep } from "node:path";
 import { shellQuote } from "../../../core/shell-quote.js";
 import { clioStateDir } from "../../../core/xdg.js";
 import {
@@ -17,6 +18,7 @@ export async function runClioRunRunner(
 	timeoutMs: number,
 	target: EvalSuiteTargetV2,
 	env?: NodeJS.ProcessEnv,
+	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
 ): Promise<EvalRunnerOutput> {
 	const prompt = runner.prompt ?? "";
 	const args = [
@@ -28,6 +30,7 @@ export async function runClioRunRunner(
 		shellQuote(target.id),
 		...(target.model === undefined ? [] : ["--model", shellQuote(target.model)]),
 		...(target.thinking === undefined ? [] : ["--thinking", shellQuote(target.thinking)]),
+		...(runner.autonomy === undefined ? [] : ["--autonomy", runner.autonomy]),
 		shellQuote(prompt),
 	];
 	const result = await runShellCommand(`${process.execPath} ${args.join(" ")}`, cwd, runner.timeoutMs ?? timeoutMs, env);
@@ -36,6 +39,7 @@ export async function runClioRunRunner(
 	const tokens = result.usage;
 	const toolMetricStream = result.metricJsonl.length > 0 ? result.metricJsonl : result.stdout;
 	const tools = toolCallMetricsFromJsonl(toolMetricStream);
+	const behavioralTools = toolBehaviorMetricEntriesFromJsonl(toolMetricStream, cwd, readObservation);
 	// Evidence metrics resolve only from the sealed receipt the --agent path
 	// prints; a runner without a receipt leaves them absent so any gate on
 	// them fails closed instead of reading prose labels.
@@ -61,6 +65,7 @@ export async function runClioRunRunner(
 			"tools.totalCalls": tools.totalCalls,
 			"tools.failed": tools.failed,
 			"tools.blocked": tools.blocked,
+			...behavioralTools,
 			"verifier.exitCode": result.exitCode,
 			...(receipt === null ? {} : evidenceMetricsFromReceipt(receipt, { envelope })),
 			...(receipt === null
@@ -70,8 +75,11 @@ export async function runClioRunRunner(
 		artifacts: {
 			stdout: result.stdout,
 			stderr: result.stderr,
+			callLedger: JSON.stringify(result.ledgerEntries),
 			...(receipt === null ? {} : { receipt: JSON.stringify(receipt) }),
 		},
+		receipt,
+		ledgerEntries: result.ledgerEntries,
 	};
 }
 
@@ -128,6 +136,125 @@ export function toolCallMetricsFromJsonl(stdout: string): ToolCallMetrics {
 	return canonicalFinishes.totalCalls > 0 ? canonicalFinishes : executionEnds;
 }
 
+interface BehavioralToolTerminal {
+	callId: string | null;
+	tool: string;
+	outcome: ToolOutcome;
+}
+
+/**
+ * Reduce the same terminal stream to bounded behavioral facts. Tool names are
+ * a dynamic metric suffix because extensions may add tools. Read paths become
+ * bounded counters against the suite's explicit public allowlist and decoy
+ * list; they never become behavioral fact values or evidence excerpts.
+ */
+export function toolBehaviorMetricEntriesFromJsonl(
+	stdout: string,
+	cwd: string,
+	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
+): Record<string, number> {
+	const starts = new Map<string, { tool: string; path: string | null }>();
+	const readPaths = new Set<string>();
+	const executionEnds: BehavioralToolTerminal[] = [];
+	const canonicalFinishes: BehavioralToolTerminal[] = [];
+	const seenExecution = new Set<string>();
+	const seenCanonical = new Set<string>();
+
+	for (const line of stdout.split(/\r?\n/)) {
+		if (line.trim().length === 0) continue;
+		let event: Record<string, unknown>;
+		try {
+			const parsed: unknown = JSON.parse(line);
+			if (!isRecord(parsed)) continue;
+			event = parsed;
+		} catch {
+			continue;
+		}
+		if (event.type === "tool_execution_start") {
+			const callId = stringField(event, "toolCallId");
+			const tool = stringField(event, "toolName");
+			if (callId === undefined || tool === undefined) continue;
+			const path = tool === "read" && isRecord(event.args) ? toolPath(event.args) : null;
+			starts.set(callId, { tool, path });
+			if (path !== null) readPaths.add(normalizeObservedPath(cwd, path));
+			continue;
+		}
+		if (event.type === "tool_execution_end") {
+			const callId = stringField(event, "toolCallId") ?? null;
+			if (callId !== null && seenExecution.has(callId)) continue;
+			if (callId !== null) seenExecution.add(callId);
+			const tool = stringField(event, "toolName") ?? (callId === null ? undefined : starts.get(callId)?.tool);
+			if (tool === undefined) continue;
+			executionEnds.push({ callId, tool, outcome: toolOutcome(event) ?? (event.isError === true ? "error" : "ok") });
+			continue;
+		}
+		if (event.type !== "clio_tool_finish" || !isRecord(event.payload)) continue;
+		const outcome = toolOutcome(event.payload);
+		const tool = stringField(event.payload, "tool");
+		if (outcome === undefined || tool === undefined) continue;
+		const callId = stringField(event.payload, "toolCallId") ?? stringField(event, "toolCallId") ?? null;
+		if (callId !== null && seenCanonical.has(callId)) continue;
+		if (callId !== null) seenCanonical.add(callId);
+		canonicalFinishes.push({ callId, tool, outcome });
+	}
+
+	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
+	const calls = new Map<string, number>();
+	const blocked = new Map<string, number>();
+	for (const terminal of terminals) {
+		const tool = metricToolName(terminal.tool);
+		calls.set(tool, (calls.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
+	}
+	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...blocked.keys()]);
+	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
+	for (const tool of [...namedTools].sort()) {
+		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
+		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
+	}
+	if (readObservation !== undefined) {
+		const allowed = readObservation.allowedPaths.map((path) => normalizeObservedPath(cwd, path));
+		const decoys = readObservation.decoyPaths.map((path) => normalizeObservedPath(cwd, path));
+		entries["tools.read.outsideAllowed"] = [...readPaths].filter(
+			(path) => !allowed.some((root) => pathWithin(path, root)),
+		).length;
+		entries["tools.read.decoyHits"] = [...readPaths].filter((path) =>
+			decoys.some((root) => pathWithin(path, root)),
+		).length;
+	}
+	return entries;
+}
+
+function toolPath(args: Record<string, unknown>): string | null {
+	for (const field of ["path", "filePath", "file_path"]) {
+		const value = args[field];
+		if (typeof value === "string" && value.length > 0 && value.length <= 4_096) return value;
+	}
+	return null;
+}
+
+function normalizeObservedPath(cwd: string, path: string): string {
+	const absolute = resolve(cwd, path);
+	const local = relative(cwd, absolute);
+	return (isAbsolute(path) && (local.startsWith("..") || isAbsolute(local)) ? absolute : local || ".")
+		.split(sep)
+		.join("/");
+}
+
+function pathWithin(path: string, root: string): boolean {
+	if (root === ".") return !isAbsolute(path) && path !== ".." && !path.startsWith("../");
+	return path === root || path.startsWith(`${root}/`);
+}
+
+function metricToolName(tool: string): string {
+	return (
+		tool
+			.toLowerCase()
+			.replaceAll(/[^a-z0-9_-]/gu, "_")
+			.slice(0, 64) || "unknown"
+	);
+}
+
 function recordToolOutcome(metrics: ToolCallMetrics, outcome: ToolOutcome): void {
 	metrics.totalCalls += 1;
 	if (outcome === "error") metrics.failed += 1;

package/src/domains/eval/runners/external-command.ts

@@ -1,5 +1,8 @@
 import { spawn } from "node:child_process";
 import { performance } from "node:perf_hooks";
+import type { RunReceipt } from "../../dispatch/types.js";
+import type { SessionEntry } from "../../session/entries.js";
+import { createEvalCallLedgerFold } from "../metrics/call-ledger-stream.js";
 import {
 	addFleetLoopObservations,
 	createFleetLoopFold,
@@ -33,6 +36,10 @@ export interface EvalRunnerOutput {
 	wallTimeMs: number;
 	metrics: Record<string, number | string | boolean | null>;
 	artifacts: Record<string, string | string[] | null>;
+	/** Parsed sealed receipt when this runner exposes one. */
+	receipt?: RunReceipt | null;
+	/** Structured per-call facts folded before bounded stdout truncation. */
+	ledgerEntries?: SessionEntry[];
 }
 
 export interface ShellCommandResult {
@@ -46,6 +53,7 @@ export interface ShellCommandResult {
 	/** Wire-stream structural invariants folded live, for the same reason. */
 	streamInvariants: EvalStreamInvariants;
 	fleetLoops: EvalFleetLoopObservation;
+	ledgerEntries: SessionEntry[];
 	stderr: string;
 	wallTimeMs: number;
 	timedOut: boolean;
@@ -75,6 +83,7 @@ export async function runExternalCommandRunner(
 	let usage = UNMEASURED_TOKEN_USAGE;
 	let streamInvariants = EMPTY_STREAM_INVARIANTS;
 	let fleetLoops = EMPTY_FLEET_LOOP_OBSERVATION;
+	const ledgerEntries: SessionEntry[] = [];
 	for (const command of commands) {
 		const result = await runShellCommand(command, cwd, runner.timeoutMs ?? timeoutMs, env);
 		stdout = appendLimited(stdout, result.stdout);
@@ -83,6 +92,7 @@ export async function runExternalCommandRunner(
 		usage = addTokenStreamUsage(usage, result.usage);
 		streamInvariants = addStreamInvariants(streamInvariants, result.streamInvariants);
 		fleetLoops = addFleetLoopObservations(fleetLoops, result.fleetLoops);
+		ledgerEntries.push(...result.ledgerEntries);
 		if (result.exitCode !== 0) {
 			return {
 				assignmentId: null,
@@ -99,6 +109,7 @@ export async function runExternalCommandRunner(
 					"verifier.exitCode": result.exitCode,
 				},
 				artifacts: {},
+				ledgerEntries,
 			};
 		}
 	}
@@ -117,6 +128,7 @@ export async function runExternalCommandRunner(
 			"verifier.exitCode": 0,
 		},
 		artifacts: {},
+		ledgerEntries,
 	};
 }
 
@@ -137,6 +149,7 @@ export function runShellCommand(
 		const usageFold = createTokenUsageFold();
 		const streamFold = createStreamInvariantFold();
 		const fleetLoopFold = createFleetLoopFold();
+		const callLedgerFold = createEvalCallLedgerFold();
 		let timedOut = false;
 		let settled = false;
 		const child = spawn(command, {
@@ -155,6 +168,7 @@ export function runShellCommand(
 			usageFold.push(chunk);
 			streamFold.push(chunk);
 			fleetLoopFold.push(chunk);
+			callLedgerFold.push(chunk);
 		});
 		child.stderr.on("data", (chunk: string) => {
 			stderr = appendLimited(stderr, chunk);
@@ -176,6 +190,7 @@ export function runShellCommand(
 				usage: usageFold.usage(),
 				streamInvariants: streamFold.invariants(),
 				fleetLoops: fleetLoopFold.observation(),
+				ledgerEntries: callLedgerFold.entries(),
 				stderr,
 				wallTimeMs: Math.round(performance.now() - started),
 				timedOut,
@@ -284,9 +299,11 @@ function compactMetricEvent(event: Record<string, unknown>): Record<string, unkn
 			toolName,
 			...(toolName === "dispatch" && isRecord(event.args)
 				? { args: event.args }
-				: toolName === "code_nav" && isRecord(event.args)
-					? { args: { mode: event.args.mode } }
-					: {}),
+				: toolName === "read" && isRecord(event.args)
+					? { args: boundedReadArgs(event.args) }
+					: toolName === "code_nav" && isRecord(event.args)
+						? { args: { mode: event.args.mode } }
+						: {}),
 		};
 	}
 	if (type === "tool_execution_end") {
@@ -313,6 +330,14 @@ function compactMetricEvent(event: Record<string, unknown>): Record<string, unkn
 	};
 }
 
+function boundedReadArgs(args: Record<string, unknown>): Record<string, string> {
+	for (const field of ["path", "filePath", "file_path"]) {
+		const value = args[field];
+		if (typeof value === "string" && value.length > 0) return { [field]: value.slice(0, 4_096) };
+	}
+	return {};
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }

package/src/domains/eval/schema/adapter.ts

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import type { EvalArtifactResultV4 } from "./artifact.js";
+import {
+	assertEvalBehaviorReferencesVerdictV1,
+	type EvalBehaviorFactSourceV1,
+	type EvalBehaviorJudgeFactV1,
+	type EvalBehaviorScenarioV1,
+	type EvalBehaviorVerdictV1,
+	judgeEvalBehaviorV1,
+} from "./behavioral.js";
+import {
+	EVAL_VERDICT_SCHEMA_V1,
+	type EvalTrackedMetricsV1,
+	type EvalVerdictEnvelopeV1,
+	parseEvalVerdictEnvelopeV1,
+} from "./verdict.js";
+
+type AdaptableSuiteResult = Pick<
+	EvalArtifactResultV4,
+	"assignmentId" | "terminalReceiptDigest" | "taskId" | "repeatIndex" | "pass" | "failureClass" | "metrics"
+>;
+
+/** Adapt one Suite v2 matrix result into the versioned verdict carried by Artifact v4. */
+export function adaptSuiteV2ResultToVerdictV1(
+	result: AdaptableSuiteResult,
+	trackedMetrics: EvalTrackedMetricsV1,
+): EvalVerdictEnvelopeV1 {
+	// The suite runner owns the one final pass decision. In particular, a
+	// declared grader failure changes `pass` without pretending the runner or
+	// its invariants broke; every other failed result is a machinery failure.
+	const machinery = result.pass || result.failureClass === "grader_failed" ? "ok" : "infrastructure_failure";
+	const outcome = result.pass ? "pass" : "fail";
+	const graderExitCode = result.metrics["task.exitCode"];
+	return parseEvalVerdictEnvelopeV1({
+		schema: EVAL_VERDICT_SCHEMA_V1,
+		scenarioId: result.taskId,
+		trialIndex: result.repeatIndex,
+		outcome,
+		machinery,
+		reason: result.pass ? null : (result.failureClass ?? "result_failed"),
+		trackedMetrics,
+		behavioral: null,
+		evidence: {
+			assignmentId: result.assignmentId,
+			terminalReceiptDigest: result.terminalReceiptDigest,
+			graderExitCode: typeof graderExitCode === "number" && Number.isInteger(graderExitCode) ? graderExitCode : null,
+		},
+	});
+}
+
+/** Adapt Suite v2 metrics into deterministic, bounded facts for a behavioral scenario. */
+export function adaptSuiteV2ResultToBehaviorV1(
+	result: AdaptableSuiteResult,
+	verdict: EvalVerdictEnvelopeV1,
+	scenario: EvalBehaviorScenarioV1,
+): EvalBehaviorVerdictV1 {
+	const requestedFacts = new Set(
+		[...scenario.expectedBehavior, ...scenario.forbiddenBehavior].map(
+			(rule) => `${rule.fact.source}\u0000${rule.fact.key}`,
+		),
+	);
+	const observedSources = new Set<EvalBehaviorFactSourceV1>();
+	const facts = Object.entries(result.metrics).flatMap(([key, value]) => {
+		if (value === null) return [];
+		const source = metricFactSource(key);
+		observedSources.add(source);
+		if (!requestedFacts.has(`${source}\u0000${key}`)) return [];
+		const serialized = JSON.stringify({ source, key, value });
+		const digest = createHash("sha256").update(serialized, "utf8").digest("hex");
+		const fact: EvalBehaviorJudgeFactV1 = {
+			id: `metric-${digest.slice(0, 16)}`,
+			source,
+			key,
+			value,
+			evidence: { locator: `artifact.metrics.${key}`, digest, excerpt: serialized.slice(0, 1_000) },
+		};
+		return [fact];
+	});
+	const allSources: EvalBehaviorFactSourceV1[] = ["transcript", "tool", "receipt", "grader"];
+	const unavailableSources = allSources.filter(
+		(source) => !observedSources.has(source) || (source === "tool" && scenario.execution.toolTarget === "none"),
+	);
+	const behavior = judgeEvalBehaviorV1(scenario, verdict, {
+		facts,
+		unavailableSources,
+		infrastructureFailure: verdict.machinery === "infrastructure_failure",
+	});
+	assertEvalBehaviorReferencesVerdictV1(behavior, verdict);
+	return behavior;
+}
+
+function metricFactSource(key: string): EvalBehaviorFactSourceV1 {
+	if (key.startsWith("tools.")) return "tool";
+	if (
+		key.startsWith("task.") ||
+		key.startsWith("claims.") ||
+		key.startsWith("completion.") ||
+		key === "result.pass" ||
+		key === "verifier.exitCode"
+	)
+		return "grader";
+	if (
+		key.startsWith("receipt.") ||
+		key.startsWith("evidence.") ||
+		key.startsWith("boundary.") ||
+		key.startsWith("loop.") ||
+		key.startsWith("cost.")
+	)
+		return "receipt";
+	return "transcript";
+}

package/src/domains/eval/schema/artifact.ts

@@ -1,4 +1,10 @@
+import type { EvalScenarioAggregateV1 } from "../metrics/aggregate.js";
 import type { EvalClioProvenance, EvalEnvironmentProvenance } from "../types.js";
+import type { EvalBehaviorVerdictV1 } from "./behavioral.js";
+import type { EvalBehaviorMetricsV1 } from "./behavioral-metrics.js";
+import type { EvalExecutionEnvelopeV1, EvalExecutionMatrixDimensionV1 } from "./execution-envelope.js";
+import type { EvalServingConfigurationV1 } from "./serving.js";
+import type { EvalVerdictEnvelopeV1 } from "./verdict.js";
 
 export interface EvalTokenMetricsV4 {
 	input: number;
@@ -45,6 +51,14 @@ export interface EvalArtifactResultV4 extends EvalArtifactAssignmentReference {
 	failureClass: string | null;
 	metrics: Record<string, number | string | boolean | null>;
 	artifacts: Record<string, string | string[] | null>;
+	/** Additive Suite v2 adapter output. Current runners always populate it. */
+	verdict?: EvalVerdictEnvelopeV1;
+	/** Optional sibling document that references the unchanged verdict v1 identity. */
+	behavioral?: EvalBehaviorVerdictV1;
+	/** Additive, typed multi-metric projection for behavioral comparison. */
+	behavioralMetrics?: EvalBehaviorMetricsV1;
+	/** Exact prompt, route, policy, and project-context identity for a behavioral result. */
+	executionEnvelope?: EvalExecutionEnvelopeV1;
 }
 
 /** The only current eval artifact format. Routing accepts this version only. */
@@ -61,8 +75,14 @@ export interface EvalArtifactV4 {
 		target: string;
 		model: string | null;
 		thinking: string | null;
+		/** Envelope fields intentionally varied by this matrix. */
+		dimensions?: EvalExecutionMatrixDimensionV1[];
 	};
+	/** Exact serving facts used to decide whether two eval runs are comparable. */
+	servingConfiguration?: EvalServingConfigurationV1;
 	summary: EvalArtifactSummaryV4;
+	/** Scenario reductions over the versioned per-trial verdicts. */
+	aggregates?: EvalScenarioAggregateV1[];
 	results: EvalArtifactResultV4[];
 }