@iowarp/clio-coder 0.4.4 → 0.4.6

package/src/domains/eval/compare/compare.ts

@@ -132,7 +132,7 @@ export function compareEvalArtifactsV4(
 export function renderEvalComparisonV4(summary: EvalCompareV4Summary): string {
 	const envelopeFailures = summary.envelopeMismatches.map(
 		(mismatch) =>
-			`  incomparable envelope: ${mismatch.scenarioId} ${mismatch.role} ${mismatch.target.id}/${mismatch.target.model ?? "none"} fields=${mismatch.fields.join(",")}`,
+			`  incomparable envelope: ${mismatch.scenarioId} ${mismatch.role} ${renderEvalComparisonRoutesV1(mismatch)} fields=${mismatch.fields.join(",")}`,
 	);
 	const affected = summary.affectedCorpusResults.map(
 		(result) =>
@@ -142,7 +142,7 @@ export function renderEvalComparisonV4(summary: EvalCompareV4Summary): string {
 	const roleReports = renderRollups("per-role baseline/candidate report", summary.roleReports);
 	const hardFailures = summary.hardGate.failures.map(
 		(failure) =>
-			`  hard failure: ${failure.scenarioId} ${failure.role} ${failure.target.id}/${failure.target.model ?? "none"} ${failure.metric} ${failure.change}`,
+			`  hard failure: ${failure.scenarioId} ${failure.role} ${renderEvalComparisonRoutesV1(failure)} ${failure.metric} ${failure.change}`,
 	);
 	const tracked = summary.trackedMetrics.flatMap((row, index) => [
 		...(index === 0
@@ -172,14 +172,13 @@ export function renderEvalComparisonV4(summary: EvalCompareV4Summary): string {
 		...(index === 0
 			? [
 					"behavioral metrics:",
-					"scenario role target model family metric baseline_mean baseline_variance baseline_coverage candidate_mean candidate_variance candidate_coverage mean_delta variance_delta change variance_change comparability gate source",
+					"scenario role baseline_target/model -> candidate_target/model family metric baseline_mean baseline_variance baseline_coverage candidate_mean candidate_variance candidate_coverage mean_delta variance_delta noise_band change variance_change comparability gate source",
 				]
 			: []),
 		[
 			row.scenarioId,
 			row.role,
-			row.target.id,
-			row.target.model ?? "none",
+			renderEvalComparisonRoutesV1(row),
 			row.family,
 			row.metric,
 			formatMetric(row.baseline.mean),
@@ -190,6 +189,7 @@ export function renderEvalComparisonV4(summary: EvalCompareV4Summary): string {
 			`${row.candidate.measured}/${row.candidate.observations}`,
 			formatSignedMetric(row.meanDelta),
 			formatSignedMetric(row.varianceDelta),
+			formatMetric(row.noiseBand),
 			row.change,
 			row.varianceChange,
 			row.comparability.comparable ? "comparable" : `incomparable:${row.comparability.mismatchedFields.join(",")}`,
@@ -218,6 +218,16 @@ export function renderEvalComparisonV4(summary: EvalCompareV4Summary): string {
 	].join("\n");
 }
 
+export function renderEvalComparisonRoutesV1(identity: {
+	target: { id: string; model: string | null };
+	baselineTargets?: ReadonlyArray<{ id: string; model: string | null }>;
+	candidateTargets?: ReadonlyArray<{ id: string; model: string | null }>;
+}): string {
+	const render = (targets: ReadonlyArray<{ id: string; model: string | null }>) =>
+		targets.map((target) => `${target.id}/${target.model ?? "none"}`).join(",") || "missing";
+	return `${render(identity.baselineTargets ?? [identity.target])} -> ${render(identity.candidateTargets ?? [identity.target])}`;
+}
+
 function behaviorRollups(
 	rows: ReadonlyArray<EvalBehaviorMetricComparisonV1>,
 	keyOf: (row: EvalBehaviorMetricComparisonV1) => string,

package/src/domains/eval/compare/envelope.ts

@@ -5,6 +5,8 @@ export interface EvalEnvelopeMismatchV1 {
 	scenarioId: string;
 	role: string;
 	target: { id: string; model: string | null };
+	baselineTargets?: Array<{ id: string; model: string | null }>;
+	candidateTargets?: Array<{ id: string; model: string | null }>;
 	fields: string[];
 }
 
@@ -20,10 +22,15 @@ export function compareEvalExecutionEnvelopesV1(
 	baselineDimensions: ReadonlyArray<EvalExecutionMatrixDimensionV1>,
 	candidateDimensions: ReadonlyArray<EvalExecutionMatrixDimensionV1>,
 ): EvalEnvelopeMismatchV1 | null {
+	const mismatchIdentity = {
+		scenarioId: identity.scenarioId,
+		role: identity.role,
+		target: { id: identity.target.id, model: identity.target.model },
+	};
 	const leftDimensions = [...baselineDimensions].sort();
 	const rightDimensions = [...candidateDimensions].sort();
 	if (stableJson(leftDimensions) !== stableJson(rightDimensions)) {
-		return { ...identity, fields: ["matrix.dimensions"] };
+		return { ...mismatchIdentity, fields: ["matrix.dimensions"] };
 	}
 	if (
 		(baseline.some((result) => result.executionEnvelope !== undefined) &&
@@ -31,22 +38,22 @@ export function compareEvalExecutionEnvelopesV1(
 		(candidate.some((result) => result.executionEnvelope !== undefined) &&
 			candidate.some((result) => result.executionEnvelope === undefined))
 	) {
-		return { ...identity, fields: ["executionEnvelope.missingTrial"] };
+		return { ...mismatchIdentity, fields: ["executionEnvelope.missingTrial"] };
 	}
 	const ignored = new Set(leftDimensions);
 	const baselineEnvelopes = uniqueEnvelopes(baseline, ignored);
 	const candidateEnvelopes = uniqueEnvelopes(candidate, ignored);
 	if (baselineEnvelopes.length === 0 && candidateEnvelopes.length === 0) return null;
 	if (baselineEnvelopes.length === 0 || candidateEnvelopes.length === 0) {
-		return { ...identity, fields: ["executionEnvelope"] };
+		return { ...mismatchIdentity, fields: ["executionEnvelope"] };
 	}
 	if (baselineEnvelopes.length > 1 || candidateEnvelopes.length > 1) {
-		return { ...identity, fields: ["executionEnvelope.withinRunVariance"] };
+		return { ...mismatchIdentity, fields: ["executionEnvelope.withinRunVariance"] };
 	}
 	const left = baselineEnvelopes[0];
 	const right = candidateEnvelopes[0];
 	if (left === undefined || right === undefined || stableJson(left) === stableJson(right)) return null;
-	return { ...identity, fields: differingFields(left, right, ignored) };
+	return { ...mismatchIdentity, fields: differingFields(left, right, ignored) };
 }
 
 function uniqueEnvelopes(

package/src/domains/eval/metrics/context.ts

@@ -1,3 +1,6 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 import { enumerateWorkspaceFiles } from "../../../core/workspace-files.js";
 import { readCodewiki, structuralCodewikiHash } from "../../context/codewiki/artifact.js";
 import { renderCodewikiDigest } from "../../context/codewiki/digest.js";
@@ -10,13 +13,31 @@ export function collectContextMetrics(cwd: string): Record<string, number | stri
 		const visiblePaths = new Set(enumerateWorkspaceFiles(cwd));
 		const artifactSourceFiles = codewiki.files.filter((file) => file.lang !== "config");
 		const indexedFiles = artifactSourceFiles.filter((file) => visiblePaths.has(file.path)).length;
-		const staleFiles = artifactSourceFiles.length - indexedFiles;
+		let contentValidFiles = 0;
+		let missingFiles = 0;
+		for (const file of artifactSourceFiles) {
+			try {
+				// Match the indexer's UTF-8 source identity, including legacy records
+				// whose missing hash cannot establish content validity.
+				const hash = createHash("sha256")
+					.update(readFileSync(join(cwd, file.path), "utf8"))
+					.digest("hex")
+					.slice(0, 16);
+				if (visiblePaths.has(file.path) && file.hash === hash) contentValidFiles += 1;
+			} catch (error) {
+				const code = (error as NodeJS.ErrnoException).code;
+				if (code === "ENOENT" || code === "ENOTDIR") missingFiles += 1;
+			}
+		}
+		const staleFiles = artifactSourceFiles.length - contentValidFiles;
 		const sourceFiles = detectProjectProfile(cwd).sourceFiles;
 		const digest = renderCodewikiDigest(codewiki);
 		return {
 			"context.indexedFiles": indexedFiles,
 			"context.artifactFiles": artifactSourceFiles.length,
 			"context.staleFiles": staleFiles,
+			"context.contentValidFiles": contentValidFiles,
+			"context.missingFiles": missingFiles,
 			"context.coverage": sourceFiles === 0 ? (artifactSourceFiles.length === 0 ? 1 : 0) : indexedFiles / sourceFiles,
 			"context.structuralHash": structuralCodewikiHash(codewiki),
 			"context.digestTokens": Math.ceil(digest.length / 4),
@@ -26,6 +47,8 @@ export function collectContextMetrics(cwd: string): Record<string, number | stri
 			"context.indexedFiles": 0,
 			"context.artifactFiles": 0,
 			"context.staleFiles": 0,
+			"context.contentValidFiles": 0,
+			"context.missingFiles": 0,
 			"context.coverage": 0,
 			"context.structuralHash": null,
 			"context.digestTokens": 0,

package/src/domains/eval/metrics/tracked.ts

@@ -1,9 +1,11 @@
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import type { RunReceipt } from "../../dispatch/types.js";
+import type { ProjectPreloadClass } from "../../prompts/preload.js";
 import { listSessionLedgerRefs, parseSessionEntries } from "../../session/archive-readers.js";
 import { extractReasoningTokens } from "../../session/context-accounting.js";
 import type { SessionEntry } from "../../session/entries.js";
+import { isProjectPreloadClass } from "../../session/prompt-manifest.js";
 import { evalHarnessMetricsFromReceipt } from "../harness-metrics.js";
 import type {
 	EvalMetricSource,
@@ -22,14 +24,7 @@ export interface EvalLedgerSnapshot {
 export interface EvalPromptManifestObservation {
 	systemPromptHash: string;
 	thinkingLevel: string | null;
-	projectPreload: {
-		mode: "full" | "synopsis" | "none";
-		chars: number;
-		lines: number;
-		reason: string | null;
-		nearLimit: boolean;
-		label: string;
-	} | null;
+	projectPreload: ProjectPreloadClass | null;
 	fragments: Array<{ id: string; contentHash: string }>;
 }
 
@@ -408,26 +403,7 @@ function promptManifestObservation(record: Record<string, unknown>): EvalPromptM
 		return contentHash === null ? [] : [{ id: entry.id, contentHash }];
 	});
 	const preload = record.projectPreload;
-	const projectPreload: EvalPromptManifestObservation["projectPreload"] =
-		preload === null
-			? null
-			: isRecord(preload) &&
-					(preload.mode === "full" || preload.mode === "synopsis" || preload.mode === "none") &&
-					typeof preload.chars === "number" &&
-					Number.isInteger(preload.chars) &&
-					typeof preload.lines === "number" &&
-					Number.isInteger(preload.lines) &&
-					typeof preload.nearLimit === "boolean" &&
-					typeof preload.label === "string"
-				? {
-						mode: preload.mode as "full" | "synopsis" | "none",
-						chars: preload.chars,
-						lines: preload.lines,
-						reason: nullableString(preload.reason),
-						nearLimit: preload.nearLimit,
-						label: preload.label,
-					}
-				: null;
+	const projectPreload = isProjectPreloadClass(preload) ? preload : null;
 	return {
 		systemPromptHash,
 		thinkingLevel: nullableString(record.thinkingLevel),

package/src/domains/eval/reports/comparison.ts

@@ -1,5 +1,5 @@
 import type { EvalCompareV4Summary } from "../compare/compare.js";
-import { renderEvalComparisonV4 } from "../compare/compare.js";
+import { renderEvalComparisonRoutesV1, renderEvalComparisonV4 } from "../compare/compare.js";
 
 export type EvalComparisonReportFormat = "text" | "json" | "md" | "junit";
 
@@ -16,7 +16,7 @@ export function renderEvalComparisonReportV1(
 function renderMarkdown(summary: EvalCompareV4Summary): string {
 	const rows = summary.behavioralMetrics.map(
 		(row) =>
-			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(`${row.target.id}/${row.target.model ?? "none"}`)} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
+			`| ${cell(row.scenarioId)} | ${cell(row.role)} | ${cell(renderEvalComparisonRoutesV1(row))} | ${row.family} | ${row.metric} | ${format(row.baseline.mean)} | ${format(row.baseline.variance)} | ${row.baseline.measured}/${row.baseline.observations} | ${format(row.candidate.mean)} | ${format(row.candidate.variance)} | ${row.candidate.measured}/${row.candidate.observations} | ${format(row.noiseBand)} | ${row.change} | ${row.varianceChange} | ${row.comparability.comparable ? "comparable" : cell(row.comparability.mismatchedFields.join(", "))} | ${row.hardGate ? "hard" : "informational"} |`,
 	);
 	const scenarioRows = summary.scenarioReports.map(
 		(report) => `| ${cell(report.id)} | ${changeCounts(report.metrics)} | ${changeCounts(report.variance)} |`,
@@ -30,11 +30,11 @@ function renderMarkdown(summary: EvalCompareV4Summary): string {
 		`Behavioral hard gate: **${summary.hardGate.pass ? "pass" : "fail"}**`,
 		...summary.hardGate.failures.map(
 			(failure) =>
-				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${failure.target.id}/${failure.target.model ?? "none"} / ${failure.metric}: ${failure.change}`,
+				`- Hard failure: ${failure.scenarioId} / ${failure.role} / ${renderEvalComparisonRoutesV1(failure)} / ${failure.metric}: ${failure.change}`,
 		),
 		...summary.envelopeMismatches.map(
 			(mismatch) =>
-				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${mismatch.target.id}/${mismatch.target.model ?? "none"}: ${mismatch.fields.join(", ")}`,
+				`- Incomparable envelope: ${mismatch.scenarioId} / ${mismatch.role} / ${renderEvalComparisonRoutesV1(mismatch)}: ${mismatch.fields.join(", ")}`,
 		),
 		...summary.affectedCorpusResults.map(
 			(result) => `- Affected corpus result: ${result.scenarioId} / ${result.role}: ${result.changedFields.join(", ")}`,
@@ -43,8 +43,8 @@ function renderMarkdown(summary: EvalCompareV4Summary): string {
 		`Token delta: ${summary.tokenDelta === null ? "unmeasured" : summary.tokenDelta}`,
 		`Wall-time delta ms: ${summary.wallTimeDelta}`,
 		"",
-		"| Scenario | Role | Target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Change | Variance | Comparability | Gate |",
-		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|",
+		"| Scenario | Role | Baseline -> candidate target/model | Family | Metric | Baseline mean | Baseline variance | Baseline measured | Candidate mean | Candidate variance | Candidate measured | Noise band | Change | Variance | Comparability | Gate |",
+		"|---|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|",
 		...rows,
 		"",
 		"## Per-scenario baseline/candidate report",
@@ -70,7 +70,7 @@ function renderJunit(summary: EvalCompareV4Summary): string {
 	);
 	const represented = new Set<string>();
 	const cases = summary.behavioralMetrics.map((row) => {
-		const name = `${row.scenarioId}[${row.role}:${row.target.id}:${row.target.model ?? "none"}].${row.metric}`;
+		const name = `${row.scenarioId}[${row.role}:${renderEvalComparisonRoutesV1(row)}].${row.metric}`;
 		const key = JSON.stringify([row.scenarioId, row.role, row.target.id, row.target.model, row.metric]);
 		represented.add(key);
 		const detail = `change=${row.change} variance=${row.varianceChange} baseline=${format(row.baseline.mean)} candidate=${format(row.candidate.mean)}`;
@@ -87,13 +87,13 @@ function renderJunit(summary: EvalCompareV4Summary): string {
 			failure.metric,
 		]);
 		if (represented.has(key)) continue;
-		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].${failure.metric}`;
+		const name = `${failure.scenarioId}[${failure.role}:${renderEvalComparisonRoutesV1(failure)}].${failure.metric}`;
 		cases.push(
 			`  <testcase classname="eval.behavior.hard" name="${escapeXml(name)}"><failure message="${escapeXml(failure.change)}">hard behavioral gate</failure></testcase>`,
 		);
 	}
 	for (const failure of summary.hardGate.envelopeFailures) {
-		const name = `${failure.scenarioId}[${failure.role}:${failure.target.id}:${failure.target.model ?? "none"}].execution-envelope`;
+		const name = `${failure.scenarioId}[${failure.role}:${renderEvalComparisonRoutesV1(failure)}].execution-envelope`;
 		cases.push(
 			`  <testcase classname="eval.behavior.envelope" name="${escapeXml(name)}"><failure message="incomparable">${escapeXml(failure.fields.join(", "))}</failure></testcase>`,
 		);

package/src/domains/eval/runners/clio-run.ts

@@ -1,5 +1,6 @@
 import { isAbsolute, relative, resolve, sep } from "node:path";
 import { normalizeClioCoderEventRecord } from "../../../core/naming-events.js";
+import { ALL_TOOL_NAMES } from "../../../core/tool-names.js";
 import { clioStateDir } from "../../../core/xdg.js";
 import {
 	evidenceMetricsFromReceipt,
@@ -205,19 +206,22 @@ export function toolBehaviorMetricEntriesFromJsonl(
 	const terminals = canonicalFinishes.length > 0 ? canonicalFinishes : executionEnds;
 	const calls = new Map<string, number>();
 	const succeeded = new Map<string, number>();
+	const failed = new Map<string, number>();
 	const blocked = new Map<string, number>();
 	for (const terminal of terminals) {
 		const tool = metricToolName(terminal.tool);
 		calls.set(tool, (calls.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "ok") succeeded.set(tool, (succeeded.get(tool) ?? 0) + 1);
 		if (terminal.outcome === "blocked") blocked.set(tool, (blocked.get(tool) ?? 0) + 1);
+		if (terminal.outcome === "error") failed.set(tool, (failed.get(tool) ?? 0) + 1);
 	}
-	const namedTools = new Set(["bash", "dispatch", "read", ...calls.keys(), ...succeeded.keys(), ...blocked.keys()]);
+	const namedTools = new Set([...ALL_TOOL_NAMES, ...calls.keys()]);
 	const entries: Record<string, number> = { "tools.read.distinctPaths": readPaths.size };
 	for (const tool of [...namedTools].sort()) {
 		entries[`tools.calls.${tool}`] = calls.get(tool) ?? 0;
 		entries[`tools.succeeded.${tool}`] = succeeded.get(tool) ?? 0;
 		entries[`tools.blocked.${tool}`] = blocked.get(tool) ?? 0;
+		entries[`tools.failed.${tool}`] = failed.get(tool) ?? 0;
 	}
 	if (readObservation !== undefined) {
 		const allowed = readObservation.allowedPaths.map((path) => normalizeObservedPath(cwd, path));

package/src/domains/eval/schema/suite.ts

@@ -230,6 +230,8 @@ export interface EvalRunnerV2 {
 }
 
 export interface EvalVerifyV2 {
+	/** Workspace-relative grader code and dependencies that the runner must not change. */
+	protectedFiles?: string[];
 	commands?: string[];
 	/**
 	 * Commands that grade the task outcome. They run in the workspace before the

package/src/domains/eval/schema/validate.ts

@@ -247,6 +247,7 @@ function readVerify(
 	return {
 		commands: readOptionalStringArray(value, "commands", `${path}.commands`, issues),
 		measure: readOptionalStringArray(value, "measure", `${path}.measure`, issues),
+		protectedFiles: readOptionalStringArray(value, "protectedFiles", `${path}.protectedFiles`, issues),
 		assertions: readAssertions(value.assertions, `${path}.assertions`, issues),
 		forbidPaths: readOptionalStringArray(value, "forbidPaths", `${path}.forbidPaths`, issues),
 	};

package/src/domains/eval/suites/run.ts

@@ -1,7 +1,9 @@
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { lstat, mkdir, mkdtemp, open, readdir, realpath, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { resolve } from "node:path";
+import { isAbsolute, relative, resolve, sep } from "node:path";
 import { namingCompatibilityEnvironment } from "../../../core/naming-compat.js";
+import { clioDataDir } from "../../../core/xdg.js";
 import { resolveMetricAssertion } from "../compare/thresholds.js";
 import { buildEvalExecutionEnvelopeV1, type EvalExecutionObservationV1 } from "../execution-provenance.js";
 import { aggregateEvalVerdicts } from "../metrics/aggregate.js";
@@ -44,7 +46,8 @@ import type { EvalMetricAssertion, EvalSuiteTargetV2, LoadedEvalSuiteV2 } from "
 import { createEvalId } from "../store.js";
 import { runCommandVerifiers } from "../verifiers/command.js";
 import { forbiddenPathHits } from "../verifiers/file-exists.js";
-import { collectPatchMetrics } from "../verifiers/patch.js";
+import { EvalGraderIntegrityError, protectGraderFiles } from "../verifiers/integrity.js";
+import { collectPatchMetrics, forgetPatchBaseline, recordPatchBaseline } from "../verifiers/patch.js";
 import { prepareGitWorkspace } from "../workspaces/git.js";
 import { type PreparedEvalWorkspace, prepareLocalWorkspace } from "../workspaces/local.js";
 import { type PrepareTempCopyWorkspaceOptions, prepareTempCopyWorkspace } from "../workspaces/temp-copy.js";
@@ -99,6 +102,7 @@ export async function runEvalSuiteV2(
 			options.clioEntry,
 			options.freshWorkspaces === true,
 			options.tempCopy,
+			resolve(clioDataDir(), "evals", evalId, "transcripts", String(results.length)),
 		);
 		spentUsd += resultCostUsd(completed.result);
 		results.push(completed.result);
@@ -154,10 +158,13 @@ async function runMatrixItem(
 	clioEntry: string,
 	freshWorkspace: boolean,
 	tempCopy: PrepareTempCopyWorkspaceOptions | undefined,
+	transcriptDir: string,
 ): Promise<CompletedMatrixItem> {
 	let workspace: PreparedEvalWorkspace | null = null;
 	let receipt: EvalRunnerOutput["receipt"] = null;
 	let runnerWallTimeMs = 0;
+	let runnerEvidence: EvalRunnerOutput | null = null;
+	let retention: Promise<RetainedSessionLedgers> | undefined;
 	let executionObservation: EvalExecutionObservationV1 | undefined;
 	// One matrix item, one Clio journal. An item measures Clio, and a shared
 	// state directory would mix sibling processes' runs and yesterday's sessions
@@ -170,20 +177,24 @@ async function runMatrixItem(
 		// A fixture that never came up measured nothing, so the item fails as a
 		// harness failure rather than reporting an invariant it never observed.
 		if (!setup.pass) throw new EvalWorkspaceSetupError(setup.exitCode, setup.stderr);
+		if (task.workspace.kind !== "local" || freshWorkspace) recordPatchBaseline(workspace.dir);
+		const checkGraderIntegrity = await protectGraderFiles(workspace.dir, task.verify.protectedFiles ?? []);
 		const runner = await runTaskRunner(task, target, workspace.dir, clioEntry, {
 			CLIO_CODER_STATE_DIR: stateDir,
 			CLIO_CODER_ENTRY: clioEntry,
 		});
-		const runnerStdoutFile = resolve(stateDir, "eval-runner-output.jsonl");
-		await writeFile(runnerStdoutFile, runner.stdout, "utf8");
+		runnerEvidence = runner;
 		receipt = runner.receipt ?? null;
 		runnerWallTimeMs = runner.wallTimeMs;
+		const runnerStdoutFile = resolve(stateDir, "eval-runner-output.jsonl");
+		await writeFile(runnerStdoutFile, runner.stdout, { encoding: "utf8", mode: 0o600, flag: "wx" });
 		const patch = collectPatchMetrics(workspace.dir);
 		const receiptExitCode = runner.exitCode;
 		// Read after the runner returned and before the journal is removed: what
 		// Clio sealed for this item, judged against its own ledger, and whether
 		// the workers it attested are still running.
 		const journalMetrics = invariantMetrics(stateDir, receiptExitCode);
+		await checkGraderIntegrity();
 		const measurement = await measureTaskOutcome(task, workspace.dir, {
 			...namingCompatibilityEnvironment(
 				"CLIO_CODER_EVAL_RUNNER_STDOUT_FILE",
@@ -202,14 +213,20 @@ async function runMatrixItem(
 			// The one reading that needs both sides: what the loop reported it
 			// spent, and what the journal shows it sealed.
 			...fleetLoopReceiptAgreement(runner.metrics as Record<string, number | boolean>, journalMetrics),
-			"patch.bytes": patch.bytes,
-			"patch.filesChanged": patch.filesChanged,
-			"patch.testFilesModified": patch.testFilesModified,
+			...(patch === null
+				? {}
+				: {
+						"patch.bytes": patch.bytes,
+						"patch.filesChanged": patch.filesChanged,
+						"patch.testFilesModified": patch.testFilesModified,
+					}),
 			"result.pass": runner.exitCode === 0,
 			"result.failureClass": runner.exitCode === 0 ? null : "runner_failed",
 			...measurement.metrics,
 		};
+		await checkGraderIntegrity();
 		const verifier = await runVerifiers(task, workspace.dir, metrics);
+		await checkGraderIntegrity();
 		const graderFailed = metrics["task.solved"] === false;
 		const pass = runner.exitCode === 0 && verifier.pass && !graderFailed;
 		const failureClass = pass
@@ -238,7 +255,10 @@ async function runMatrixItem(
 				...(verifier.stderr.length > 0 ? { verifierStderr: verifier.stderr } : {}),
 			},
 		};
-		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		retention ??= retainSessionLedgers(stateDir, transcriptDir, runnerEvidence?.stdout);
+		const retained = await retention;
+		applyRetainedSessionLedgers(result, retained);
+		const snapshot = retained.errors.length > 0 ? emptyLedgerSnapshot() : await readEvalLedgerSnapshot(stateDir);
 		const selected = selectEvalLedgerEntries(snapshot.entries, runner.ledgerEntries ?? []);
 		result.artifacts.trackedMetricSources = JSON.stringify({
 			assistantCalls: selected.source,
@@ -271,10 +291,15 @@ async function runMatrixItem(
 			serving: evalServingObservationFrom(target, receipt ?? null, snapshot.compiledPromptHashes),
 		};
 	} catch (error) {
-		const failureClass = error instanceof EvalWorkspaceSetupError ? "setup_failed" : "command_error";
+		const failureClass =
+			error instanceof EvalGraderIntegrityError
+				? "grader_integrity"
+				: error instanceof EvalWorkspaceSetupError
+					? "setup_failed"
+					: "command_error";
 		const result: EvalArtifactResultV4 = {
-			assignmentId: null,
-			terminalReceiptDigest: null,
+			assignmentId: runnerEvidence?.assignmentId ?? null,
+			terminalReceiptDigest: runnerEvidence?.terminalReceiptDigest ?? null,
 			taskId: task.id,
 			repeatIndex,
 			target: { id: target.id, model: target.model ?? null, thinking: target.thinking ?? null },
@@ -287,11 +312,15 @@ async function runMatrixItem(
 				"latency.wallMs": 0,
 			},
 			artifacts: {
+				...runnerEvidence?.artifacts,
 				error: error instanceof Error ? error.message : String(error),
 				...(workspace === null ? {} : { workspace: workspace.dir }),
 			},
 		};
-		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		retention ??= retainSessionLedgers(stateDir, transcriptDir, runnerEvidence?.stdout);
+		const retained = await retention;
+		applyRetainedSessionLedgers(result, retained);
+		const snapshot = retained.errors.length > 0 ? emptyLedgerSnapshot() : await readEvalLedgerSnapshot(stateDir);
 		result.verdict = adaptSuiteV2ResultToVerdictV1(
 			result,
 			buildEvalTrackedMetrics({
@@ -316,6 +345,7 @@ async function runMatrixItem(
 		};
 	} finally {
 		try {
+			if (workspace !== null) forgetPatchBaseline(workspace.dir);
 			await workspace?.cleanup();
 		} finally {
 			// A workspace cleanup error must not strand the suite-owned journal.
@@ -575,3 +605,149 @@ function nullableNonNegativeInteger(value: unknown): number | null {
 function stringArray(value: unknown): string[] {
 	return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : [];
 }
+
+interface RetainedSessionLedgers {
+	runnerStdoutPath?: string;
+	paths: string[];
+	errors: string[];
+}
+
+function applyRetainedSessionLedgers(result: EvalArtifactResultV4, retained: RetainedSessionLedgers): void {
+	if (retained.runnerStdoutPath !== undefined) result.artifacts.runnerStdoutFile = retained.runnerStdoutPath;
+	if (retained.paths.length > 0 || retained.errors.length > 0) result.artifacts.sessionLedgers = retained.paths;
+	if (retained.errors.length === 0) return;
+	result.artifacts.sessionLedgerErrors = retained.errors;
+	result.metrics["evidence.retentionComplete"] = false;
+	// Keep a runner/integrity failure as the primary cause; an otherwise healthy
+	// task outcome cannot claim successful grading with unsafe or missing evidence.
+	if (result.pass || result.failureClass === "grader_failed") {
+		result.artifacts.outcomeBeforeEvidenceFailure = result.failureClass ?? "pass";
+		result.pass = false;
+		result.failureClass = "evidence_integrity";
+		result.metrics["result.pass"] = false;
+		result.metrics["result.failureClass"] = "evidence_integrity";
+	}
+}
+
+/** Reject symlinks at every component inside the owned source directory. */
+async function plainLedgerPath(root: string, path: string): Promise<void> {
+	const rel = relative(root, path);
+	if (isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) throw new Error("unsafe ledger path");
+	let current = root;
+	for (const part of ["", ...rel.split(sep)]) {
+		current = resolve(current, part);
+		if ((await lstat(current)).isSymbolicLink()) throw new Error("unsafe ledger symlink");
+	}
+}
+
+async function readOwnedLedger(root: string, path: string): Promise<Buffer> {
+	await plainLedgerPath(root, path);
+	const before = await lstat(path);
+	if (!before.isFile()) throw new Error("ledger is not a regular file");
+	const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+	try {
+		const opened = await file.stat();
+		await plainLedgerPath(root, path);
+		if (opened.dev !== before.dev || opened.ino !== before.ino) throw new Error("ledger changed while opening");
+		return await file.readFile();
+	} finally {
+		await file.close();
+	}
+}
+
+/** Create a private, unique capture below the operator's artifact root. */
+async function transcriptCaptureDirectory(destination: string): Promise<string> {
+	const base = clioDataDir();
+	await mkdir(base, { recursive: true, mode: 0o700 });
+	let current = await realpath(base);
+	const rel = relative(base, destination);
+	if (isAbsolute(rel) || rel === ".." || rel.startsWith(`..${sep}`)) throw new Error("unsafe transcript destination");
+	for (const part of rel.split(sep)) {
+		current = resolve(current, part);
+		try {
+			await mkdir(current, { mode: 0o700 });
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+		}
+		const info = await lstat(current);
+		if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("unsafe transcript destination");
+	}
+	return mkdtemp(resolve(current, "capture-"));
+}
+
+/** Enumerate only ordinary directories; never traverse a runner-created alias. */
+async function discoverOwnedLedgers(stateDir: string, errors: string[]): Promise<string[]> {
+	const root = resolve(stateDir, "sessions");
+	try {
+		await plainLedgerPath(stateDir, root);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+		throw error;
+	}
+	const paths: string[] = [];
+	for (const cwd of await readdir(root, { withFileTypes: true })) {
+		if (cwd.isSymbolicLink()) {
+			errors.push("session directory rejected: symlink");
+			continue;
+		}
+		if (!cwd.isDirectory()) continue;
+		const cwdPath = resolve(root, cwd.name);
+		try {
+			await plainLedgerPath(stateDir, cwdPath);
+			for (const session of await readdir(cwdPath, { withFileTypes: true })) {
+				if (session.isSymbolicLink()) {
+					errors.push("session directory rejected: symlink");
+					continue;
+				}
+				if (session.isDirectory()) paths.push(resolve(cwdPath, session.name, "current.jsonl"));
+			}
+		} catch {
+			errors.push("session directory unreadable or unsafe");
+		}
+	}
+	return paths.sort();
+}
+
+/** Best-effort evidence collection never throws into (or retries) the failure path. */
+async function retainSessionLedgers(
+	stateDir: string,
+	destination: string,
+	runnerStdout?: string,
+): Promise<RetainedSessionLedgers> {
+	const retained: RetainedSessionLedgers = { paths: [], errors: [] };
+	try {
+		const refs = await discoverOwnedLedgers(stateDir, retained.errors);
+		if (refs.length === 0 && !runnerStdout) return retained;
+		const directory = await transcriptCaptureDirectory(destination);
+		if (runnerStdout) {
+			const path = resolve(directory, "runner-stdout.jsonl");
+			await writeFile(path, runnerStdout, { mode: 0o600, flag: "wx" });
+			retained.runnerStdoutPath = path;
+		}
+		for (const [index, pathToLedger] of refs.entries()) {
+			try {
+				let content: Buffer;
+				try {
+					content = await readOwnedLedger(stateDir, pathToLedger);
+				} catch (error) {
+					if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+					content = await readOwnedLedger(stateDir, `${pathToLedger}.tmp`);
+					retained.errors.push(`ledger ${index}: incomplete current ledger; temporary evidence retained`);
+				}
+				const path = resolve(directory, `${index}.jsonl`);
+				await writeFile(path, content, { mode: 0o600, flag: "wx" });
+				retained.paths.push(path);
+			} catch (error) {
+				// Error codes describe missing/unreadable/unsafe evidence without echoing
+				// runner-controlled paths or external file contents into the artifact.
+				const code = (error as NodeJS.ErrnoException).code;
+				retained.errors.push(
+					`ledger ${index}: retention failed (${code === "ENOENT" ? "missing" : "unsafe or unreadable"})`,
+				);
+			}
+		}
+	} catch {
+		retained.errors.push("session ledger retention failed: artifact storage unavailable or unsafe");
+	}
+	return retained;
+}