@iowarp/clio-coder 0.4.3 → 0.4.5

package/src/domains/eval/metrics/tracked.ts

@@ -1,9 +1,11 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import type { RunReceipt } from "../../dispatch/types.js";
+import type { ProjectPreloadClass } from "../../prompts/preload.js";
 import { listSessionLedgerRefs, parseSessionEntries } from "../../session/archive-readers.js";
 import { extractReasoningTokens } from "../../session/context-accounting.js";
 import type { SessionEntry } from "../../session/entries.js";
+import { isProjectPreloadClass } from "../../session/prompt-manifest.js";
 import { evalHarnessMetricsFromReceipt } from "../harness-metrics.js";
 import type {
 	EvalMetricSource,
@@ -22,14 +24,7 @@ export interface EvalLedgerSnapshot {
 export interface EvalPromptManifestObservation {
 	systemPromptHash: string;
 	thinkingLevel: string | null;
-	projectPreload: {
-		mode: "full" | "synopsis" | "none";
-		chars: number;
-		lines: number;
-		reason: string | null;
-		nearLimit: boolean;
-		label: string;
-	} | null;
+	projectPreload: ProjectPreloadClass | null;
 	fragments: Array<{ id: string; contentHash: string }>;
 }
 
@@ -408,26 +403,7 @@ function promptManifestObservation(record: Record<string, unknown>): EvalPromptM
 		return contentHash === null ? [] : [{ id: entry.id, contentHash }];
 	});
 	const preload = record.projectPreload;
-	const projectPreload: EvalPromptManifestObservation["projectPreload"] =
-		preload === null
-			? null
-			: isRecord(preload) &&
-					(preload.mode === "full" || preload.mode === "synopsis" || preload.mode === "none") &&
-					typeof preload.chars === "number" &&
-					Number.isInteger(preload.chars) &&
-					typeof preload.lines === "number" &&
-					Number.isInteger(preload.lines) &&
-					typeof preload.nearLimit === "boolean" &&
-					typeof preload.label === "string"
-				? {
-						mode: preload.mode as "full" | "synopsis" | "none",
-						chars: preload.chars,
-						lines: preload.lines,
-						reason: nullableString(preload.reason),
-						nearLimit: preload.nearLimit,
-						label: preload.label,
-					}
-				: null;
+	const projectPreload = isProjectPreloadClass(preload) ? preload : null;
 	return {
 		systemPromptHash,
 		thinkingLevel: nullableString(record.thinkingLevel),

package/src/domains/eval/reports/comparison.ts

@@ -1,5 +1,5 @@
 import type { EvalCompareV4Summary } from "../compare/compare.js";
-import { renderEvalComparisonV4 } from "../compare/compare.js";
+import { renderEvalComparisonRoutesV1, renderEvalComparisonV4 } from "../compare/compare.js";
 
 export type EvalComparisonReportFormat = "text" | "json" | "md" | "junit";
 
@@ -16,7 +16,7 @@ export function renderEvalComparisonReportV1(
 function renderMarkdown(summary: EvalCompareV4Summary): string {
 	const rows = summary.behavioralMetrics.map(
 		(row) =>
-			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(`${row.target.id}/${row.target.model ?? "none"}`)} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
+			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(renderEvalComparisonRoutesV1(row))} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${format(row.noiseBand)} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
 	);
 	const scenarioRows = summary.scenarioReports.map(
 		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
@@ -30,11 +30,11 @@ function renderMarkdown(summary: EvalCompareV4Summary): string {
 		`Behavioral hard gate: **${summary.hardGate.pass ? "pass" : "fail"}**`,
 		...summary.hardGate.failures.map(
 			(failure) =>
-				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${failure.target.id}/${failure.target.model ?? "none"} / ${failure.metric}: ${failure.change}`,
+				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${renderEvalComparisonRoutesV1(failure)} / ${failure.metric}: ${failure.change}`,
 		),
 		...summary.envelopeMismatches.map(
 			(mismatch) =>
-				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${mismatch.target.id}/${mismatch.target.model ?? "none"}: ${mismatch.fields.join(", ")}`,
+				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${renderEvalComparisonRoutesV1(mismatch)}: ${mismatch.fields.join(", ")}`,
 		),
 		...summary.affectedCorpusResults.map(
 			(result) => `- Affected corpus result: ${result.scenarioId} / ${result.role}: ${result.changedFields.join(", ")}`,
@@ -43,8 +43,8 @@ function renderMarkdown(summary: EvalCompareV4Summary): string {
 		`Token delta: ${summary.tokenDelta === null ? "unmeasured" : summary.tokenDelta}`,
 		`Wall-time delta ms: ${summary.wallTimeDelta}`,
 		"",
-		"| Scenario | Role | Target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Change | Variance | Comparability | Gate |",
-		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|",
+		"| Scenario | Role | Baseline -> candidate target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Noise band | Change | Variance | Comparability | Gate |",
+		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|",
 		...rows,
 		"",
 		"## Per-scenario baseline/candidate report",
@@ -70,7 +70,7 @@ function renderJunit(summary: EvalCompareV4Summary): string {
 	);
 	const represented = new Set<string>();
 	const cases = summary.behavioralMetrics.map((row) => {
-		const name = `${row.scenarioId}[${row.role}:${row.target.id}:${row.target.model ?? "none"}].${row.metric}`;
+		const name = `${row.scenarioId}[${row.role}:${renderEvalComparisonRoutesV1(row)}].${row.metric}`;
 		const key = JSON.stringify([row.scenarioId, row.role, row.target.id, row.target.model, row.metric]);
 		represented.add(key);
 		const detail = `change=${row.change} variance=${row.varianceChange} baseline=${format(row.baseline.mean)} candidate=${format(row.candidate.mean)}`;
@@ -87,13 +87,13 @@ function renderJunit(summary: EvalCompareV4Summary): string {
 			failure.metric,
 		]);
 		if (represented.has(key)) continue;
-		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].${failure.metric}`;
+		const name = `${failure.scenarioId}[${failure.role}:${renderEvalComparisonRoutesV1(failure)}].${failure.metric}`;
 		cases.push(
 			`  <testcase classname="eval.behavior.hard" name="${escapeXml(name)}"><failure message="${escapeXml(failure.change)}">hard behavioral gate</failure></testcase>`,
 		);
 	}
 	for (const failure of summary.hardGate.envelopeFailures) {
-		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].execution-envelope`;
+		const name = `${failure.scenarioId}[${failure.role}:${renderEvalComparisonRoutesV1(failure)}].execution-envelope`;
 		cases.push(
 			`  <testcase classname="eval.behavior.envelope" name="${escapeXml(name)}"><failure message="incomparable">${escapeXml(failure.fields.join(", "))}</failure></testcase>`,
 		);

package/src/domains/eval/runners/clio-run.ts

@@ -1,6 +1,6 @@
 import { isAbsolute, relative, resolve, sep } from "node:path";
 import { normalizeClioCoderEventRecord } from "../../../core/naming-events.js";
-import { shellQuote } from "../../../core/shell-quote.js";
+import { ALL_TOOL_NAMES } from "../../../core/tool-names.js";
 import { clioStateDir } from "../../../core/xdg.js";
 import {
 	evidenceMetricsFromReceipt,
@@ -22,19 +22,23 @@ export async function runClioRunRunner(
 	readObservation?: { allowedPaths: string[]; decoyPaths: string[] },
 ): Promise<EvalRunnerOutput> {
 	const prompt = runner.prompt ?? "";
-	const args = [
-		shellQuote(clioEntry),
+	// Spawned as an argv, not a shell line: the task timeout has to reach the
+	// Clio process itself so the receipt it seals and the exit code this
+	// runner reports describe the same ending. See runShellCommand.
+	const args: [string, ...string[]] = [
+		process.execPath,
+		clioEntry,
 		"run",
 		"--json",
-		...(runner.agent === undefined ? [] : ["--agent", shellQuote(runner.agent)]),
+		...(runner.agent === undefined ? [] : ["--agent", runner.agent]),
 		"--target",
-		shellQuote(target.id),
-		...(target.model === undefined ? [] : ["--model", shellQuote(target.model)]),
-		...(target.thinking === undefined ? [] : ["--thinking", shellQuote(target.thinking)]),
+		target.id,
+		...(target.model === undefined ? [] : ["--model", target.model]),
+		...(target.thinking === undefined ? [] : ["--thinking", target.thinking]),
 		...(runner.autonomy === undefined ? [] : ["--autonomy", runner.autonomy]),
-		shellQuote(prompt),
+		prompt,
 	];
-	const result = await runShellCommand(`${process.execPath} ${args.join(" ")}`, cwd, runner.timeoutMs ?? timeoutMs, env);
+	const result = await runShellCommand(args, cwd, runner.timeoutMs ?? timeoutMs, env);
 	// Usage is folded from the live stream, not from the bounded stdout
 	// artifact: a verbose run's `message_end` events do not survive truncation.
 	const tokens = result.usage;
@@ -202,19 +206,22 @@ export function toolBehaviorMetricEntriesFromJsonl(
 	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
 	const calls = new Map<string, number>();
 	const succeeded = new Map<string, number>();
+	const failed = new Map<string, number>();
 	const blocked = new Map<string, number>();
 	for (const terminal of terminals) {
 		const tool = metricToolName(terminal.tool);
 		calls.set(tool, (calls.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "ok") succeeded.set(tool, (succeeded.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "error") failed.set(tool, (failed.get(tool) ?? 0) + 1);
 	}
-	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...succeeded.keys(), ...blocked.keys()]);
+	const namedTools = new Set([...ALL_TOOL_NAMES, ...calls.keys()]);
 	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
 	for (const tool of [...namedTools].sort()) {
 		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
 		entries[`tools.succeeded.${tool}`] = succeeded.get(tool) ?? 0;
 		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
+		entries[`tools.failed.${tool}`] = failed.get(tool) ?? 0;
 	}
 	if (readObservation !== undefined) {
 		const allowed = readObservation.allowedPaths.map((path) => normalizeObservedPath(cwd, path));

package/src/domains/eval/runners/context-index.ts

@@ -1,4 +1,3 @@
-import { shellQuote } from "../../../core/shell-quote.js";
 import type { EvalSuiteTargetV2 } from "../schema/suite.js";
 import { type EvalRunnerOutput, runShellCommand } from "./external-command.js";
 
@@ -9,12 +8,8 @@ export async function runContextIndexRunner(
 	target: EvalSuiteTargetV2,
 	env?: NodeJS.ProcessEnv,
 ): Promise<EvalRunnerOutput> {
-	const result = await runShellCommand(
-		`${shellQuote(process.execPath)} ${shellQuote(clioEntry)} context index --json`,
-		cwd,
-		timeoutMs,
-		env,
-	);
+	const args: [string, ...string[]] = [process.execPath, clioEntry, "context", "index", "--json"];
+	const result = await runShellCommand(args, cwd, timeoutMs, env);
 	const parsed = parseContextIndexOutput(result.stdout);
 	return {
 		assignmentId: null,

package/src/domains/eval/runners/context-init.ts

@@ -1,6 +1,5 @@
 import { existsSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { shellQuote } from "../../../core/shell-quote.js";
 import type { EvalRunnerV2, EvalSuiteTargetV2 } from "../schema/suite.js";
 import { type EvalRunnerOutput, runShellCommand } from "./external-command.js";
 
@@ -13,7 +12,7 @@ export async function runContextInitRunner(
 	env?: NodeJS.ProcessEnv,
 ): Promise<EvalRunnerOutput> {
 	const extraArgs = runner.args ?? [];
-	const command = [
+	const args: [string, ...string[]] = [
 		process.execPath,
 		clioEntry,
 		"context",
@@ -25,10 +24,8 @@ export async function runContextInitRunner(
 		...(target.model === undefined ? [] : ["--model", target.model]),
 		...(target.thinking === undefined ? [] : ["--thinking", target.thinking]),
 		...extraArgs,
-	]
-		.map(shellQuote)
-		.join(" ");
-	const result = await runShellCommand(command, cwd, runner.timeoutMs ?? timeoutMs, env);
+	];
+	const result = await runShellCommand(args, cwd, runner.timeoutMs ?? timeoutMs, env);
 	const payload = parseInitPayload(result.stdout);
 	const candidateGeneration = recordField(payload, "generation");
 	const generation = isValidGenerationPayload(payload, candidateGeneration) ? candidateGeneration : null;

package/src/domains/eval/runners/external-command.ts

@@ -65,6 +65,11 @@ const OUTPUT_HEAD_LIMIT = 20_000;
 const OUTPUT_TRUNCATION_MARKER = "\n[output middle truncated; tail preserved]\n";
 const METRIC_JSONL_LIMIT = 256_000;
 const METRIC_JSONL_LINE_LIMIT = 64_000;
+// After the deadline's SIGTERM, how long a run gets to end on its own terms
+// before SIGKILL. Clio seals the interrupted run's receipt inside its
+// coordinated shutdown, which budgets 500ms per hook across several hooks; a
+// one-second grace killed it mid-seal and the timed-out run left no receipt.
+const TIMEOUT_KILL_GRACE_MS = 5_000;
 
 export async function runExternalCommandRunner(
 	runner: EvalRunnerV2,
@@ -133,8 +138,23 @@ export async function runExternalCommandRunner(
 	};
 }
 
+/**
+ * Run one eval subprocess to completion or to the timeout.
+ *
+ * A string runs through the shell, which is what an operator's
+ * `external-command` line and the verifier commands are. An argv array spawns
+ * the program directly, with no shell between the runner and the process
+ * whose exit code it reports. That distinction matters on the timeout path:
+ * `/bin/sh -c` on dash keeps the shell as the parent, so SIGTERM to the shell
+ * stops the shell alone while the program runs on to completion. A
+ * `clio-coder run` spawned that way finished its turn after the deadline,
+ * sealed a succeeded receipt with exit 0, and exited 0, and this runner
+ * reported the shell's signal death as exit 124 (issue #275). With an argv the
+ * signal reaches Clio itself, which seals the run as canceled with the exit
+ * status it then reports, so the receipt and the exit code agree either way.
+ */
 export function runShellCommand(
-	command: string,
+	command: string | readonly [string, ...string[]],
 	cwd: string,
 	timeoutMs: number,
 	env?: NodeJS.ProcessEnv,
@@ -153,14 +173,12 @@ export function runShellCommand(
 		const callLedgerFold = createEvalCallLedgerFold();
 		let timedOut = false;
 		let settled = false;
-		const child = spawn(command, {
-			cwd,
-			shell: true,
-			stdio: ["ignore", "pipe", "pipe"],
-			// The overlay is additive: an eval item pins where Clio writes its
-			// journal, and inherits everything else the operator's shell provides.
-			env: env === undefined ? process.env : { ...process.env, ...env },
-		});
+		// The overlay is additive: an eval item pins where Clio writes its
+		// journal, and inherits everything else the operator's shell provides.
+		const spawnEnv = env === undefined ? process.env : { ...process.env, ...env };
+		const [file, args, shell] = typeof command === "string" ? [command, [], true] : [command[0], command.slice(1), false];
+		const child = spawn(file, args, { cwd, shell, stdio: ["ignore", "pipe", "pipe"], env: spawnEnv });
+		const commandLine = typeof command === "string" ? command : command.join(" ");
 		child.stdout.setEncoding("utf8");
 		child.stderr.setEncoding("utf8");
 		child.stdout.on("data", (chunk: string) => {
@@ -177,14 +195,14 @@ export function runShellCommand(
 		const timer = setTimeout(() => {
 			timedOut = true;
 			child.kill("SIGTERM");
-			setTimeout(() => child.kill("SIGKILL"), 1000);
+			setTimeout(() => child.kill("SIGKILL"), TIMEOUT_KILL_GRACE_MS);
 		}, timeoutMs);
 		const finish = (exitCode: number): void => {
 			if (settled) return;
 			settled = true;
 			clearTimeout(timer);
 			resolve({
-				command,
+				command: commandLine,
 				exitCode,
 				stdout,
 				metricJsonl: metricCapture.finish(),

package/src/domains/eval/schema/suite.ts

@@ -230,6 +230,8 @@ export interface EvalRunnerV2 {
 }
 
 export interface EvalVerifyV2 {
+	/** Workspace-relative grader code and dependencies that the runner must not change. */
+	protectedFiles?: string[];
 	commands?: string[];
 	/**
 	 * Commands that grade the task outcome. They run in the workspace before the

package/src/domains/eval/schema/validate.ts

@@ -247,6 +247,7 @@ function readVerify(
 	return {
 		commands: readOptionalStringArray(value, "commands", `${path}.commands`, issues),
 		measure: readOptionalStringArray(value, "measure", `${path}.measure`, issues),
+		protectedFiles: readOptionalStringArray(value, "protectedFiles", `${path}.protectedFiles`, issues),
 		assertions: readAssertions(value.assertions, `${path}.assertions`, issues),
 		forbidPaths: readOptionalStringArray(value, "forbidPaths", `${path}.forbidPaths`, issues),
 	};

package/src/domains/eval/suites/run.ts

@@ -1,7 +1,9 @@
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { lstat, mkdir, mkdtemp, open, readdir, realpath, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { resolve } from "node:path";
+import { isAbsolute, relative, resolve, sep } from "node:path";
 import { namingCompatibilityEnvironment } from "../../../core/naming-compat.js";
+import { clioDataDir } from "../../../core/xdg.js";
 import { resolveMetricAssertion } from "../compare/thresholds.js";
 import { buildEvalExecutionEnvelopeV1, type EvalExecutionObservationV1 } from "../execution-provenance.js";
 import { aggregateEvalVerdicts } from "../metrics/aggregate.js";
@@ -44,7 +46,8 @@ import type { EvalMetricAssertion, EvalSuiteTargetV2, LoadedEvalSuiteV2 } from "
 import { createEvalId } from "../store.js";
 import { runCommandVerifiers } from "../verifiers/command.js";
 import { forbiddenPathHits } from "../verifiers/file-exists.js";
-import { collectPatchMetrics } from "../verifiers/patch.js";
+import { EvalGraderIntegrityError, protectGraderFiles } from "../verifiers/integrity.js";
+import { collectPatchMetrics, forgetPatchBaseline, recordPatchBaseline } from "../verifiers/patch.js";
 import { prepareGitWorkspace } from "../workspaces/git.js";
 import { type PreparedEvalWorkspace, prepareLocalWorkspace } from "../workspaces/local.js";
 import { type PrepareTempCopyWorkspaceOptions, prepareTempCopyWorkspace } from "../workspaces/temp-copy.js";
@@ -99,6 +102,7 @@ export async function runEvalSuiteV2(
 			options.clioEntry,
 			options.freshWorkspaces === true,
 			options.tempCopy,
+			resolve(clioDataDir(), "evals", evalId, "transcripts", String(results.length)),
 		);
 		spentUsd += resultCostUsd(completed.result);
 		results.push(completed.result);
@@ -154,10 +158,13 @@ async function runMatrixItem(
 	clioEntry: string,
 	freshWorkspace: boolean,
 	tempCopy: PrepareTempCopyWorkspaceOptions | undefined,
+	transcriptDir: string,
 ): Promise<CompletedMatrixItem> {
 	let workspace: PreparedEvalWorkspace | null = null;
 	let receipt: EvalRunnerOutput["receipt"] = null;
 	let runnerWallTimeMs = 0;
+	let runnerEvidence: EvalRunnerOutput | null = null;
+	let retention: Promise<RetainedSessionLedgers> | undefined;
 	let executionObservation: EvalExecutionObservationV1 | undefined;
 	// One matrix item, one Clio journal. An item measures Clio, and a shared
 	// state directory would mix sibling processes' runs and yesterday's sessions
@@ -170,20 +177,24 @@ async function runMatrixItem(
 		// A fixture that never came up measured nothing, so the item fails as a
 		// harness failure rather than reporting an invariant it never observed.
 		if (!setup.pass) throw new EvalWorkspaceSetupError(setup.exitCode, setup.stderr);
+		if (task.workspace.kind !== "local" || freshWorkspace) recordPatchBaseline(workspace.dir);
+		const checkGraderIntegrity = await protectGraderFiles(workspace.dir, task.verify.protectedFiles ?? []);
 		const runner = await runTaskRunner(task, target, workspace.dir, clioEntry, {
 			CLIO_CODER_STATE_DIR: stateDir,
 			CLIO_CODER_ENTRY: clioEntry,
 		});
-		const runnerStdoutFile = resolve(stateDir, "eval-runner-output.jsonl");
-		await writeFile(runnerStdoutFile, runner.stdout, "utf8");
+		runnerEvidence = runner;
 		receipt = runner.receipt ?? null;
 		runnerWallTimeMs = runner.wallTimeMs;
+		const runnerStdoutFile = resolve(stateDir, "eval-runner-output.jsonl");
+		await writeFile(runnerStdoutFile, runner.stdout, { encoding: "utf8", mode: 0o600, flag: "wx" });
 		const patch = collectPatchMetrics(workspace.dir);
 		const receiptExitCode = runner.exitCode;
 		// Read after the runner returned and before the journal is removed: what
 		// Clio sealed for this item, judged against its own ledger, and whether
 		// the workers it attested are still running.
 		const journalMetrics = invariantMetrics(stateDir, receiptExitCode);
+		await checkGraderIntegrity();
 		const measurement = await measureTaskOutcome(task, workspace.dir, {
 			...namingCompatibilityEnvironment(
 				"CLIO_CODER_EVAL_RUNNER_STDOUT_FILE",
@@ -202,14 +213,20 @@ async function runMatrixItem(
 			// The one reading that needs both sides: what the loop reported it
 			// spent, and what the journal shows it sealed.
 			...fleetLoopReceiptAgreement(runner.metrics as Record<string, number | boolean>, journalMetrics),
-			"patch.bytes": patch.bytes,
-			"patch.filesChanged": patch.filesChanged,
-			"patch.testFilesModified": patch.testFilesModified,
+			...(patch === null
+				? {}
+				: {
+						"patch.bytes": patch.bytes,
+						"patch.filesChanged": patch.filesChanged,
+						"patch.testFilesModified": patch.testFilesModified,
+					}),
 			"result.pass": runner.exitCode === 0,
 			"result.failureClass": runner.exitCode === 0 ? null : "runner_failed",
 			...measurement.metrics,
 		};
+		await checkGraderIntegrity();
 		const verifier = await runVerifiers(task, workspace.dir, metrics);
+		await checkGraderIntegrity();
 		const graderFailed = metrics["task.solved"] === false;
 		const pass = runner.exitCode === 0 && verifier.pass && !graderFailed;
 		const failureClass = pass
@@ -238,7 +255,10 @@ async function runMatrixItem(
 				...(verifier.stderr.length > 0 ? { verifierStderr: verifier.stderr } : {}),
 			},
 		};
-		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		retention ??= retainSessionLedgers(stateDir, transcriptDir, runnerEvidence?.stdout);
+		const retained = await retention;
+		applyRetainedSessionLedgers(result, retained);
+		const snapshot = retained.errors.length > 0 ? emptyLedgerSnapshot() : await readEvalLedgerSnapshot(stateDir);
 		const selected = selectEvalLedgerEntries(snapshot.entries, runner.ledgerEntries ?? []);
 		result.artifacts.trackedMetricSources = JSON.stringify({
 			assistantCalls: selected.source,
@@ -271,10 +291,15 @@ async function runMatrixItem(
 			serving: evalServingObservationFrom(target, receipt ?? null, snapshot.compiledPromptHashes),
 		};
 	} catch (error) {
-		const failureClass = error instanceof EvalWorkspaceSetupError ? "setup_failed" : "command_error";
+		const failureClass =
+			error instanceof EvalGraderIntegrityError
+				? "grader_integrity"
+				: error instanceof EvalWorkspaceSetupError
+					? "setup_failed"
+					: "command_error";
 		const result: EvalArtifactResultV4 = {
-			assignmentId: null,
-			terminalReceiptDigest: null,
+			assignmentId: runnerEvidence?.assignmentId ?? null,
+			terminalReceiptDigest: runnerEvidence?.terminalReceiptDigest ?? null,
 			taskId: task.id,
 			repeatIndex,
 			target: { id: target.id, model: target.model ?? null, thinking: target.thinking ?? null },
@@ -287,11 +312,15 @@ async function runMatrixItem(
 				"latency.wallMs": 0,
 			},
 			artifacts: {
+				...runnerEvidence?.artifacts,
 				error: error instanceof Error ? error.message : String(error),
 				...(workspace === null ? {} : { workspace: workspace.dir }),
 			},
 		};
-		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		retention ??= retainSessionLedgers(stateDir, transcriptDir, runnerEvidence?.stdout);
+		const retained = await retention;
+		applyRetainedSessionLedgers(result, retained);
+		const snapshot = retained.errors.length > 0 ? emptyLedgerSnapshot() : await readEvalLedgerSnapshot(stateDir);
 		result.verdict = adaptSuiteV2ResultToVerdictV1(
 			result,
 			buildEvalTrackedMetrics({
@@ -316,6 +345,7 @@ async function runMatrixItem(
 		};
 	} finally {
 		try {
+			if (workspace !== null) forgetPatchBaseline(workspace.dir);
 			await workspace?.cleanup();
 		} finally {
 			// A workspace cleanup error must not strand the suite-owned journal.
@@ -575,3 +605,149 @@ function nullableNonNegativeInteger(value: unknown): number | null {
 function stringArray(value: unknown): string[] {
 	return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : [];
 }
+
+interface RetainedSessionLedgers {
+	runnerStdoutPath?: string;
+	paths: string[];
+	errors: string[];
+}
+
+function applyRetainedSessionLedgers(result: EvalArtifactResultV4, retained: RetainedSessionLedgers): void {
+	if (retained.runnerStdoutPath !== undefined) result.artifacts.runnerStdoutFile = retained.runnerStdoutPath;
+	if (retained.paths.length > 0 || retained.errors.length > 0) result.artifacts.sessionLedgers = retained.paths;
+	if (retained.errors.length === 0) return;
+	result.artifacts.sessionLedgerErrors = retained.errors;
+	result.metrics["evidence.retentionComplete"] = false;
+	// Keep a runner/integrity failure as the primary cause; an otherwise healthy
+	// task outcome cannot claim successful grading with unsafe or missing evidence.
+	if (result.pass || result.failureClass === "grader_failed") {
+		result.artifacts.outcomeBeforeEvidenceFailure = result.failureClass ?? "pass";
+		result.pass = false;
+		result.failureClass = "evidence_integrity";
+		result.metrics["result.pass"] = false;
+		result.metrics["result.failureClass"] = "evidence_integrity";
+	}
+}
+
+/** Reject symlinks at every component inside the owned source directory. */
+async function plainLedgerPath(root: string, path: string): Promise<void> {
+	const rel = relative(root, path);
+	if (isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) throw new Error("unsafe ledger path");
+	let current = root;
+	for (const part of ["", ...rel.split(sep)]) {
+		current = resolve(current, part);
+		if ((await lstat(current)).isSymbolicLink()) throw new Error("unsafe ledger symlink");
+	}
+}
+
+async function readOwnedLedger(root: string, path: string): Promise<Buffer> {
+	await plainLedgerPath(root, path);
+	const before = await lstat(path);
+	if (!before.isFile()) throw new Error("ledger is not a regular file");
+	const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+	try {
+		const opened = await file.stat();
+		await plainLedgerPath(root, path);
+		if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("ledger changed while opening");
+		return await file.readFile();
+	} finally {
+		await file.close();
+	}
+}
+
+/** Create a private, unique capture below the operator's artifact root. */
+async function transcriptCaptureDirectory(destination: string): Promise<string> {
+	const base = clioDataDir();
+	await mkdir(base, { recursive: true, mode: 0o700 });
+	let current = await realpath(base);
+	const rel = relative(base, destination);
+	if (isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) throw new Error("unsafe transcript destination");
+	for (const part of rel.split(sep)) {
+		current = resolve(current, part);
+		try {
+			await mkdir(current, { mode: 0o700 });
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+		}
+		const info = await lstat(current);
+		if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("unsafe transcript destination");
+	}
+	return mkdtemp(resolve(current, "capture-"));
+}
+
+/** Enumerate only ordinary directories; never traverse a runner-created alias. */
+async function discoverOwnedLedgers(stateDir: string, errors: string[]): Promise<string[]> {
+	const root = resolve(stateDir, "sessions");
+	try {
+		await plainLedgerPath(stateDir, root);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+		throw error;
+	}
+	const paths: string[] = [];
+	for (const cwd of await readdir(root, { withFileTypes: true })) {
+		if (cwd.isSymbolicLink()) {
+			errors.push("session directory rejected: symlink");
+			continue;
+		}
+		if (!cwd.isDirectory()) continue;
+		const cwdPath = resolve(root, cwd.name);
+		try {
+			await plainLedgerPath(stateDir, cwdPath);
+			for (const session of await readdir(cwdPath, { withFileTypes: true })) {
+				if (session.isSymbolicLink()) {
+					errors.push("session directory rejected: symlink");
+					continue;
+				}
+				if (session.isDirectory()) paths.push(resolve(cwdPath, session.name, "current.jsonl"));
+			}
+		} catch {
+			errors.push("session directory unreadable or unsafe");
+		}
+	}
+	return paths.sort();
+}
+
+/** Best-effort evidence collection never throws into (or retries) the failure path. */
+async function retainSessionLedgers(
+	stateDir: string,
+	destination: string,
+	runnerStdout?: string,
+): Promise<RetainedSessionLedgers> {
+	const retained: RetainedSessionLedgers = { paths: [], errors: [] };
+	try {
+		const refs = await discoverOwnedLedgers(stateDir, retained.errors);
+		if (refs.length === 0 && !runnerStdout) return retained;
+		const directory = await transcriptCaptureDirectory(destination);
+		if (runnerStdout) {
+			const path = resolve(directory, "runner-stdout.jsonl");
+			await writeFile(path, runnerStdout, { mode: 0o600, flag: "wx" });
+			retained.runnerStdoutPath = path;
+		}
+		for (const [index, pathToLedger] of refs.entries()) {
+			try {
+				let content: Buffer;
+				try {
+					content = await readOwnedLedger(stateDir, pathToLedger);
+				} catch (error) {
+					if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+					content = await readOwnedLedger(stateDir, `${pathToLedger}.tmp`);
+					retained.errors.push(`ledger ${index}: incomplete current ledger; temporary evidence retained`);
+				}
+				const path = resolve(directory, `${index}.jsonl`);
+				await writeFile(path, content, { mode: 0o600, flag: "wx" });
+				retained.paths.push(path);
+			} catch (error) {
+				// Error codes describe missing/unreadable/unsafe evidence without echoing
+				// runner-controlled paths or external file contents into the artifact.
+				const code = (error as NodeJS.ErrnoException).code;
+				retained.errors.push(
+					`ledger ${index}: retention failed (${code === "ENOENT" ? "missing" : "unsafe or unreadable"})`,
+				);
+			}
+		}
+	} catch {
+		retained.errors.push("session ledger retention failed: artifact storage unavailable or unsafe");
+	}
+	return retained;
+}