@iowarp/clio-coder 0.3.8 → 0.3.9

package/src/domains/eval/suites/run.ts

@@ -1,7 +1,9 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
 import { resolveMetricAssertion } from "../compare/thresholds.js";
+import { buildEvalExecutionEnvelopeV1, type EvalExecutionObservationV1 } from "../execution-provenance.js";
+import { aggregateEvalVerdicts } from "../metrics/aggregate.js";
 import { collectContextMetrics } from "../metrics/context.js";
 import { fleetLoopReceiptAgreement } from "../metrics/fleet-loop-stream.js";
 import {
@@ -15,12 +17,22 @@ import {
 import { wallTimeMetric } from "../metrics/latency.js";
 import { tokenAccountingFrom } from "../metrics/tokens.js";
 import { zeroToolCallMetrics } from "../metrics/tool-calls.js";
-import { evalClioProvenance, evalEnvironmentProvenance } from "../provenance.js";
+import { buildEvalTrackedMetrics, emptyEvalTrackedMetrics, readEvalLedgerSnapshot } from "../metrics/tracked.js";
+import {
+	type EvalServingObservation,
+	evalClioProvenance,
+	evalEnvironmentProvenance,
+	evalServingConfiguration,
+	evalServingObservationFrom,
+} from "../provenance.js";
 import { runClioRunRunner } from "../runners/clio-run.js";
 import { runContextIndexRunner } from "../runners/context-index.js";
 import { runContextInitRunner } from "../runners/context-init.js";
 import { type EvalRunnerOutput, runExternalCommandRunner } from "../runners/external-command.js";
+import { adaptSuiteV2ResultToBehaviorV1, adaptSuiteV2ResultToVerdictV1 } from "../schema/adapter.js";
 import type { EvalArtifactResultV4, EvalArtifactV4 } from "../schema/artifact.js";
+import { buildEvalBehaviorMetricsV1 } from "../schema/behavioral-metrics.js";
+import type { EvalServingConfigurationV1 } from "../schema/serving.js";
 import type { EvalMetricAssertion, EvalSuiteTargetV2, LoadedEvalSuiteV2 } from "../schema/suite.js";
 import { createEvalId } from "../store.js";
 import { runCommandVerifiers } from "../verifiers/command.js";
@@ -28,7 +40,7 @@ import { forbiddenPathHits } from "../verifiers/file-exists.js";
 import { collectPatchMetrics } from "../verifiers/patch.js";
 import { prepareGitWorkspace } from "../workspaces/git.js";
 import { type PreparedEvalWorkspace, prepareLocalWorkspace } from "../workspaces/local.js";
-import { prepareTempCopyWorkspace } from "../workspaces/temp-copy.js";
+import { type PrepareTempCopyWorkspaceOptions, prepareTempCopyWorkspace } from "../workspaces/temp-copy.js";
 import { expandEvalMatrix } from "./matrix.js";
 import { artifactMatrixIdentity } from "./resolve.js";
 
@@ -42,6 +54,15 @@ class EvalWorkspaceSetupError extends Error {
 export interface RunEvalSuiteV2Options {
 	clioEntry: string;
 	now?: () => Date;
+	/** Convert local workspaces into isolated copies for explicit trial runs. */
+	freshWorkspaces?: boolean;
+	/** Dependency/root seam for deterministic workspace lifecycle contracts. */
+	tempCopy?: PrepareTempCopyWorkspaceOptions;
+}
+
+interface CompletedMatrixItem {
+	result: EvalArtifactResultV4;
+	serving: EvalServingObservation;
 }
 
 export async function runEvalSuiteV2(
@@ -52,6 +73,7 @@ export async function runEvalSuiteV2(
 	const started = now();
 	const evalId = createEvalId(started, loaded.hash);
 	const results: EvalArtifactResultV4[] = [];
+	const servingObservations: EvalServingObservation[] = [];
 	const maxCostUsd = loaded.suite.matrix.maxCostUsd;
 	let spentUsd = 0;
 	for (const item of expandEvalMatrix(loaded.suite)) {
@@ -59,14 +81,24 @@ export async function runEvalSuiteV2(
 		// exceeds it, remaining items fail closed instead of running. A live
 		// suite can therefore never keep spending past its declared budget.
 		if (maxCostUsd !== undefined && spentUsd > maxCostUsd) {
-			results.push(budgetExhaustedResult(item.task.id, item.target, item.repeatIndex, spentUsd, maxCostUsd));
+			results.push(budgetExhaustedResult(loaded, item.task, item.target, item.repeatIndex, spentUsd, maxCostUsd));
 			continue;
 		}
-		const result = await runMatrixItem(loaded, item.task, item.target, item.repeatIndex, options.clioEntry);
-		spentUsd += resultCostUsd(result);
-		results.push(result);
+		const completed = await runMatrixItem(
+			loaded,
+			item.task,
+			item.target,
+			item.repeatIndex,
+			options.clioEntry,
+			options.freshWorkspaces === true,
+			options.tempCopy,
+		);
+		spentUsd += resultCostUsd(completed.result);
+		results.push(completed.result);
+		servingObservations.push(completed.serving);
 	}
-	return buildArtifact(loaded, evalId, results, options.clioEntry);
+	const serving = await evalServingConfiguration(loaded.suite.matrix.targets, servingObservations);
+	return buildArtifact(loaded, evalId, results, options.clioEntry, serving);
 }
 
 /** Known receipt cost of one finished matrix item; unpriced runs count zero. */
@@ -76,16 +108,17 @@ export function resultCostUsd(result: Pick<EvalArtifactResultV4, "metrics">): nu
 }
 
 function budgetExhaustedResult(
-	taskId: string,
+	loaded: LoadedEvalSuiteV2,
+	task: LoadedEvalSuiteV2["suite"]["tasks"][number],
 	target: EvalSuiteTargetV2,
 	repeatIndex: number,
 	spentUsd: number,
 	maxCostUsd: number,
 ): EvalArtifactResultV4 {
-	return {
+	const result: EvalArtifactResultV4 = {
 		assignmentId: null,
 		terminalReceiptDigest: null,
-		taskId,
+		taskId: task.id,
 		repeatIndex,
 		target: { id: target.id, model: target.model ?? null, thinking: target.thinking ?? null },
 		pass: false,
@@ -100,6 +133,10 @@ function budgetExhaustedResult(
 			error: `matrix cost budget exhausted: spent $${spentUsd.toFixed(4)} of max $${maxCostUsd.toFixed(4)} before this item`,
 		},
 	};
+	result.verdict = adaptSuiteV2ResultToVerdictV1(result, emptyEvalTrackedMetrics());
+	attachBehavioralResult(result, task);
+	attachExecutionEnvelope(result, task, target, loaded.baseDir, null, emptyLedgerSnapshot());
+	return result;
 }
 
 async function runMatrixItem(
@@ -108,15 +145,20 @@ async function runMatrixItem(
 	target: EvalSuiteTargetV2,
 	repeatIndex: number,
 	clioEntry: string,
-): Promise<EvalArtifactResultV4> {
+	freshWorkspace: boolean,
+	tempCopy: PrepareTempCopyWorkspaceOptions | undefined,
+): Promise<CompletedMatrixItem> {
 	let workspace: PreparedEvalWorkspace | null = null;
+	let receipt: EvalRunnerOutput["receipt"] = null;
+	let runnerWallTimeMs = 0;
+	let executionObservation: EvalExecutionObservationV1 | undefined;
 	// One matrix item, one Clio journal. An item measures Clio, and a shared
 	// state directory would mix sibling processes' runs and yesterday's sessions
 	// into the reading; pinning it here also means an item leaves nothing behind
 	// in the operator's own state.
-	const stateDir = await mkdtemp(resolve(tmpdir(), "clio-eval-state-"));
+	const stateDir = await mkdtemp(resolve(tempCopy?.tempRoot ?? tmpdir(), "clio-eval-state-"));
 	try {
-		workspace = await prepareWorkspace(loaded.baseDir, task);
+		workspace = await prepareWorkspace(loaded.baseDir, task, freshWorkspace, tempCopy);
 		const setup = await runCommandVerifiers(task.workspace.setup ?? [], workspace.dir, task.timeoutMs);
 		// A fixture that never came up measured nothing, so the item fails as a
 		// harness failure rather than reporting an invariant it never observed.
@@ -125,12 +167,20 @@ async function runMatrixItem(
 			CLIO_CODER_STATE_DIR: stateDir,
 			CLIO_CODER_ENTRY: clioEntry,
 		});
+		const runnerStdoutFile = resolve(stateDir, "eval-runner-output.jsonl");
+		await writeFile(runnerStdoutFile, runner.stdout, "utf8");
+		receipt = runner.receipt ?? null;
+		runnerWallTimeMs = runner.wallTimeMs;
 		const patch = collectPatchMetrics(workspace.dir);
 		const receiptExitCode = runner.exitCode;
 		// Read after the runner returned and before the journal is removed: what
 		// Clio sealed for this item, judged against its own ledger, and whether
 		// the workers it attested are still running.
 		const journalMetrics = invariantMetrics(stateDir, receiptExitCode);
+		const measurement = await measureTaskOutcome(task, workspace.dir, {
+			CLIO_EVAL_RUNNER_STDOUT_FILE: runnerStdoutFile,
+		});
+		executionObservation = measurement.executionObservation;
 		const metrics: Record<string, number | string | boolean | null> = {
 			...zeroToolCallMetrics(),
 			...collectContextMetrics(workspace.dir),
@@ -146,15 +196,22 @@ async function runMatrixItem(
 			"patch.testFilesModified": patch.testFilesModified,
 			"result.pass": runner.exitCode === 0,
 			"result.failureClass": runner.exitCode === 0 ? null : "runner_failed",
-			...(await measureTaskOutcome(task, workspace.dir)),
+			...measurement.metrics,
 		};
 		const verifier = await runVerifiers(task, workspace.dir, metrics);
-		const pass = runner.exitCode === 0 && verifier.pass;
-		const failureClass = pass ? null : runner.exitCode !== 0 ? "runner_failed" : verifier.failureClass;
+		const graderFailed = metrics["task.solved"] === false;
+		const pass = runner.exitCode === 0 && verifier.pass && !graderFailed;
+		const failureClass = pass
+			? null
+			: runner.exitCode !== 0
+				? "runner_failed"
+				: !verifier.pass
+					? verifier.failureClass
+					: "grader_failed";
 		metrics["verifier.exitCode"] = verifier.exitCode;
 		metrics["result.pass"] = pass;
 		metrics["result.failureClass"] = failureClass;
-		return {
+		const result: EvalArtifactResultV4 = {
 			assignmentId: runner.assignmentId,
 			terminalReceiptDigest: runner.terminalReceiptDigest,
 			taskId: task.id,
@@ -165,13 +222,30 @@ async function runMatrixItem(
 			metrics,
 			artifacts: {
 				...runner.artifacts,
+				workspace: workspace.dir,
 				...(verifier.stdout.length > 0 ? { verifierStdout: verifier.stdout } : {}),
 				...(verifier.stderr.length > 0 ? { verifierStderr: verifier.stderr } : {}),
 			},
 		};
+		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		const ledgerEntries = [...snapshot.entries, ...(runner.ledgerEntries ?? [])];
+		result.verdict = adaptSuiteV2ResultToVerdictV1(
+			result,
+			buildEvalTrackedMetrics({
+				ledgerEntries,
+				receipt: receipt ?? null,
+				fallbackWallClockMs: runner.wallTimeMs,
+			}),
+		);
+		attachBehavioralResult(result, task);
+		attachExecutionEnvelope(result, task, target, workspace.dir, receipt ?? null, snapshot, executionObservation);
+		return {
+			result,
+			serving: evalServingObservationFrom(target, receipt ?? null, snapshot.compiledPromptHashes),
+		};
 	} catch (error) {
 		const failureClass = error instanceof EvalWorkspaceSetupError ? "setup_failed" : "command_error";
-		return {
+		const result: EvalArtifactResultV4 = {
 			assignmentId: null,
 			terminalReceiptDigest: null,
 			taskId: task.id,
@@ -185,14 +259,74 @@ async function runMatrixItem(
 				"verifier.exitCode": 1,
 				"latency.wallMs": 0,
 			},
-			artifacts: { error: error instanceof Error ? error.message : String(error) },
+			artifacts: {
+				error: error instanceof Error ? error.message : String(error),
+				...(workspace === null ? {} : { workspace: workspace.dir }),
+			},
+		};
+		const snapshot = await readEvalLedgerSnapshot(stateDir);
+		result.verdict = adaptSuiteV2ResultToVerdictV1(
+			result,
+			buildEvalTrackedMetrics({
+				ledgerEntries: snapshot.entries,
+				receipt: receipt ?? null,
+				fallbackWallClockMs: runnerWallTimeMs,
+			}),
+		);
+		attachBehavioralResult(result, task);
+		attachExecutionEnvelope(
+			result,
+			task,
+			target,
+			workspace?.dir ?? loaded.baseDir,
+			receipt ?? null,
+			snapshot,
+			executionObservation,
+		);
+		return {
+			result,
+			serving: evalServingObservationFrom(target, receipt ?? null, snapshot.compiledPromptHashes),
 		};
 	} finally {
-		await workspace?.cleanup();
-		await rm(stateDir, { recursive: true, force: true });
+		try {
+			await workspace?.cleanup();
+		} finally {
+			// A workspace cleanup error must not strand the suite-owned journal.
+			await rm(stateDir, { recursive: true, force: true });
+		}
 	}
 }
 
+function attachBehavioralResult(result: EvalArtifactResultV4, task: LoadedEvalSuiteV2["suite"]["tasks"][number]): void {
+	if (task.behavioral === undefined || result.verdict === undefined) return;
+	result.behavioral = adaptSuiteV2ResultToBehaviorV1(result, result.verdict, task.behavioral);
+	result.behavioralMetrics = buildEvalBehaviorMetricsV1(result, task.behavioral.execution.subject.role);
+}
+
+function attachExecutionEnvelope(
+	result: EvalArtifactResultV4,
+	task: LoadedEvalSuiteV2["suite"]["tasks"][number],
+	target: EvalSuiteTargetV2,
+	cwd: string | null,
+	receipt: NonNullable<EvalRunnerOutput["receipt"]> | null,
+	ledger: Awaited<ReturnType<typeof readEvalLedgerSnapshot>>,
+	observation?: EvalExecutionObservationV1,
+): void {
+	if (task.behavioral === undefined) return;
+	result.executionEnvelope = buildEvalExecutionEnvelopeV1({
+		task,
+		target,
+		cwd,
+		receipt,
+		ledger,
+		...(observation === undefined ? {} : { observation }),
+	});
+}
+
+function emptyLedgerSnapshot(): Awaited<ReturnType<typeof readEvalLedgerSnapshot>> {
+	return { entries: [], compiledPromptHashes: [], promptManifests: [], contextSnapshots: [] };
+}
+
 /** Journal-derived invariants for one finished item, read from its isolated state directory. */
 function invariantMetrics(stateDir: string, runnerExitCode: number): Record<string, number | boolean> {
 	const journal = readRunJournal(stateDir);
@@ -208,10 +342,15 @@ function invariantMetrics(stateDir: string, runnerExitCode: number): Record<stri
 async function prepareWorkspace(
 	baseDir: string,
 	task: LoadedEvalSuiteV2["suite"]["tasks"][number],
+	freshWorkspace: boolean,
+	tempCopy: PrepareTempCopyWorkspaceOptions | undefined,
 ): Promise<PreparedEvalWorkspace> {
+	if (task.workspace.kind === "local" && freshWorkspace) {
+		return prepareTempCopyWorkspace(baseDir, { ...task.workspace, kind: "temp-copy" }, tempCopy);
+	}
 	if (task.workspace.kind === "local") return prepareLocalWorkspace(baseDir, task.workspace);
 	if (task.workspace.kind === "git") return prepareGitWorkspace(task.workspace);
-	return prepareTempCopyWorkspace(baseDir, task.workspace);
+	return prepareTempCopyWorkspace(baseDir, task.workspace, tempCopy);
 }
 
 async function runTaskRunner(
@@ -225,23 +364,87 @@ async function runTaskRunner(
 	if (task.runner.kind === "context-index") return runContextIndexRunner(cwd, clioEntry, task.timeoutMs, target, env);
 	if (task.runner.kind === "context-init")
 		return runContextInitRunner(task.runner, cwd, clioEntry, task.timeoutMs, target, env);
-	return runClioRunRunner(task.runner, cwd, clioEntry, task.timeoutMs, target, env);
+	return runClioRunRunner(task.runner, cwd, clioEntry, task.timeoutMs, target, env, task.metrics.readObservation);
 }
 
 /**
- * Run the task-outcome commands and report what they found. A nonzero exit is
- * recorded, never raised: whether the model solved the workload is a
- * measurement, and only the machinery's behavior is a gate. A task that
- * declares no measure commands reports nothing rather than a passing zero.
+ * Run the task-outcome grader and report what it found. A nonzero exit is
+ * recorded rather than thrown so the verdict can report a failed outcome with
+ * healthy machinery. A task that declares no measure commands reports nothing
+ * and leaves its final pass decision to the runner and gating verifiers.
  */
 async function measureTaskOutcome(
 	task: LoadedEvalSuiteV2["suite"]["tasks"][number],
 	cwd: string,
-): Promise<Record<string, number | boolean>> {
+	env?: NodeJS.ProcessEnv,
+): Promise<{ metrics: Record<string, number | boolean>; executionObservation?: EvalExecutionObservationV1 }> {
 	const commands = task.verify.measure ?? [];
-	if (commands.length === 0) return {};
-	const result = await runCommandVerifiers(commands, cwd, task.timeoutMs);
-	return { "task.exitCode": result.exitCode, "task.solved": result.exitCode === 0 };
+	if (commands.length === 0) return { metrics: {} };
+	const result = await runCommandVerifiers(commands, cwd, task.timeoutMs, env);
+	const behavioral = graderBehaviorMeasurement(result.stdout);
+	return {
+		metrics: {
+			"task.exitCode": result.exitCode,
+			"task.solved": result.exitCode === 0,
+			...behavioral.metrics,
+		},
+		...(behavioral.executionObservation === undefined ? {} : { executionObservation: behavioral.executionObservation }),
+	};
+}
+
+function graderBehaviorMeasurement(stdout: string): {
+	metrics: Record<string, number | boolean>;
+	executionObservation?: EvalExecutionObservationV1;
+} {
+	const metrics: Record<string, number | boolean> = {};
+	let executionObservation: EvalExecutionObservationV1 | undefined;
+	for (const line of stdout.split(/\r?\n/u)) {
+		if (line.trim().length === 0) continue;
+		let value: unknown;
+		try {
+			value = JSON.parse(line) as unknown;
+		} catch {
+			continue;
+		}
+		if (!isRecord(value)) continue;
+		if (value.schema === "clio.eval.measure.v1" && isRecord(value.metrics)) {
+			for (const [key, metric] of Object.entries(value.metrics)) {
+				if (key !== "claims.unsupported" && key !== "completion.reported") continue;
+				if (typeof metric === "boolean" || (typeof metric === "number" && Number.isFinite(metric))) metrics[key] = metric;
+			}
+		}
+		if (value.schema === "clio.eval.execution-observation.v1") {
+			executionObservation = parseExecutionObservation(value);
+		}
+	}
+	return { metrics, ...(executionObservation === undefined ? {} : { executionObservation }) };
+}
+
+function parseExecutionObservation(value: Record<string, unknown>): EvalExecutionObservationV1 {
+	const policies = isRecord(value.policyHashes) ? value.policyHashes : {};
+	const project = isRecord(value.projectContext) ? value.projectContext : null;
+	return {
+		compositionHash: nullableDigest(value.compositionHash),
+		target: nullableString(value.target),
+		wireModel: nullableString(value.wireModel),
+		runtime: nullableString(value.runtime),
+		thinkingLevel: nullableString(value.thinkingLevel),
+		toolSignature: nullableDigest(value.toolSignature),
+		autonomy: nullableString(value.autonomy),
+		policyHashes: { rulePack: nullableDigest(policies.rulePack), project: nullableDigest(policies.project) },
+		projectContext:
+			project === null
+				? null
+				: {
+						tier: nullableString(project.tier),
+						contentHash: nullableDigest(project.contentHash),
+						chars: nullableNonNegativeInteger(project.chars),
+						sections: stringArray(project.sections),
+						rulesApplied: stringArray(project.rulesApplied),
+						operatorProfileApplied:
+							typeof project.operatorProfileApplied === "boolean" ? project.operatorProfileApplied : null,
+					},
+	};
 }
 
 async function runVerifiers(
@@ -293,6 +496,7 @@ function buildArtifact(
 	evalId: string,
 	results: EvalArtifactResultV4[],
 	clioEntry: string,
+	servingConfiguration: EvalServingConfigurationV1,
 ): EvalArtifactV4 {
 	const passed = results.filter((result) => result.pass).length;
 	return {
@@ -301,7 +505,11 @@ function buildArtifact(
 		suite: { id: loaded.suite.suite.id, hash: loaded.hash },
 		clio: evalClioProvenance({ entry: clioEntry }),
 		environment: evalEnvironmentProvenance(),
-		matrix: artifactMatrixIdentity(loaded.suite.matrix.targets),
+		matrix: {
+			...artifactMatrixIdentity(loaded.suite.matrix.targets),
+			...(loaded.suite.matrix.dimensions === undefined ? {} : { dimensions: loaded.suite.matrix.dimensions }),
+		},
+		servingConfiguration,
 		summary: {
 			runs: results.length,
 			passed,
@@ -310,6 +518,9 @@ function buildArtifact(
 			tokens: tokenAccountingFrom(results),
 			wallTimeMs: results.reduce((sum, result) => sum + wallTimeMetric(result.metrics), 0),
 		},
+		aggregates: aggregateEvalVerdicts(
+			results.flatMap((result) => (result.verdict === undefined ? [] : [result.verdict])),
+		),
 		results,
 	};
 }
@@ -317,3 +528,23 @@ function buildArtifact(
 function assertionMessage(assertion: EvalMetricAssertion, actual: number | string | boolean | null): string {
 	return `assertion failed: ${assertion.metric} ${assertion.op} ${String(assertion.value)} (actual ${JSON.stringify(actual)})`;
 }
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function nullableString(value: unknown): string | null {
+	return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function nullableDigest(value: unknown): string | null {
+	return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value) ? value : null;
+}
+
+function nullableNonNegativeInteger(value: unknown): number | null {
+	return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null;
+}
+
+function stringArray(value: unknown): string[] {
+	return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : [];
+}

package/src/domains/eval/verifiers/command.ts

@@ -12,12 +12,13 @@ export async function runCommandVerifiers(
 	commands: ReadonlyArray<string>,
 	cwd: string,
 	timeoutMs: number,
+	env?: NodeJS.ProcessEnv,
 ): Promise<CommandVerifierResult> {
 	let stdout = "";
 	let stderr = "";
 	let wallTimeMs = 0;
 	for (const command of commands) {
-		const result = await runShellCommand(command, cwd, timeoutMs);
+		const result = await runShellCommand(command, cwd, timeoutMs, env);
 		stdout += result.stdout;
 		stderr += result.stderr;
 		wallTimeMs += result.wallTimeMs;

package/src/domains/eval/workspaces/temp-copy.ts

@@ -1,29 +1,161 @@
-import { cp, mkdtemp, rm } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { cp, lstat, mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { relative, resolve } from "node:path";
+import { promisify } from "node:util";
 import type { EvalWorkspaceV2 } from "../schema/suite.js";
 import type { PreparedEvalWorkspace } from "./local.js";
 
+const execFileAsync = promisify(execFile);
+const GIT_FILE_LIST_LIMIT_BYTES = 128 * 1024 * 1024;
+
+export interface EvalWorkspaceCopyOptions {
+	recursive: true;
+	filter: (source: string, destination: string) => boolean;
+}
+
+export type EvalWorkspaceCopy = (
+	source: string,
+	destination: string,
+	options: EvalWorkspaceCopyOptions,
+) => Promise<void>;
+
+export interface PrepareTempCopyWorkspaceOptions {
+	/** Test/integration seam; production uses node:fs/promises.cp. */
+	copy?: EvalWorkspaceCopy;
+	/** Parent for both workspace and suite-owned state directories. */
+	tempRoot?: string;
+}
+
 export async function prepareTempCopyWorkspace(
 	baseDir: string,
 	workspace: EvalWorkspaceV2,
+	options: PrepareTempCopyWorkspaceOptions = {},
 ): Promise<PreparedEvalWorkspace> {
 	const source = resolve(baseDir, workspace.path ?? ".");
-	const dest = await mkdtemp(resolve(tmpdir(), "clio-eval-workspace-"));
-	const excludes = workspace.excludes ?? [];
-	await cp(source, dest, {
-		recursive: true,
-		filter: (path) => !isExcluded(relative(source, path), excludes),
-	});
-	return {
-		dir: dest,
-		cleanup: async () => {
-			await rm(dest, { recursive: true, force: true });
-		},
-	};
+	const dest = await mkdtemp(resolve(options.tempRoot ?? tmpdir(), "clio-eval-workspace-"));
+	try {
+		const selection = await gitCopySelection(source);
+		const excludes = workspace.excludes ?? [];
+		const copyWorkspace = options.copy ?? defaultCopy;
+		await copyWorkspace(source, dest, {
+			recursive: true,
+			filter: (path) => shouldCopy(relative(source, path), excludes, selection),
+		});
+		return {
+			dir: dest,
+			cleanup: async () => {
+				await rm(dest, { recursive: true, force: true });
+			},
+		};
+	} catch (error) {
+		// `mkdtemp` succeeded before either Git enumeration or the recursive
+		// copy could throw. The preparer still owns that incomplete destination.
+		await rm(dest, { recursive: true, force: true });
+		throw error;
+	}
 }
 
 function isExcluded(rel: string, excludes: ReadonlyArray<string>): boolean {
 	const normalized = rel.replaceAll("\\", "/");
 	return excludes.some((entry) => normalized === entry || normalized.startsWith(`${entry.replaceAll("\\", "/")}/`));
 }
+
+async function defaultCopy(source: string, destination: string, options: EvalWorkspaceCopyOptions): Promise<void> {
+	await cp(source, destination, options);
+}
+
+interface GitCopySelection {
+	files: ReadonlySet<string>;
+	directories: ReadonlySet<string>;
+}
+
+function shouldCopy(
+	relativePath: string,
+	excludes: ReadonlyArray<string>,
+	selection: GitCopySelection | null,
+): boolean {
+	const normalized = relativePath.replaceAll("\\", "/");
+	if (normalized.length === 0) return true;
+	if (isExcluded(normalized, excludes)) return false;
+	if (selection === null) return true;
+	return selection.files.has(normalized) || selection.directories.has(normalized);
+}
+
+/**
+ * A Git checkout supplies the authoritative copy set: tracked files plus
+ * untracked files that are not ignored. Outside Git, null preserves the
+ * existing recursive-copy behavior.
+ */
+async function gitCopySelection(source: string): Promise<GitCopySelection | null> {
+	let inside: string;
+	try {
+		inside = await gitOutput(source, ["rev-parse", "--is-inside-work-tree"]);
+	} catch (error) {
+		// Outside a checkout, Git's refusal selects the legacy recursive path. A
+		// checkout whose Git query broke must fail closed: falling back there can
+		// silently copy the ignored data this branch exists to exclude.
+		if (await hasGitMarker(source)) throw error;
+		return null;
+	}
+	if (inside.trim() !== "true") return null;
+
+	const output = await gitOutput(source, [
+		"--literal-pathspecs",
+		"ls-files",
+		"-z",
+		"--cached",
+		"--others",
+		"--exclude-standard",
+		"--",
+		".",
+	]);
+	const files = new Set<string>();
+	const directories = new Set<string>();
+	for (const path of output.split("\0")) {
+		if (path.length === 0) continue;
+		const normalized = normalizeGitPath(path);
+		if (normalized === null) throw new Error("git ls-files returned a path outside the eval workspace");
+		files.add(normalized);
+		let separator = normalized.lastIndexOf("/");
+		while (separator >= 0) {
+			directories.add(normalized.slice(0, separator));
+			separator = normalized.lastIndexOf("/", separator - 1);
+		}
+	}
+	return { files, directories };
+}
+
+async function gitOutput(cwd: string, args: ReadonlyArray<string>): Promise<string> {
+	const { stdout } = await execFileAsync("git", [...args], {
+		cwd,
+		encoding: "utf8",
+		maxBuffer: GIT_FILE_LIST_LIMIT_BYTES,
+	});
+	return stdout;
+}
+
+function normalizeGitPath(path: string): string | null {
+	const normalized = path.replaceAll("\\", "/").replace(/^\.\//u, "");
+	if (normalized.length === 0 || normalized.startsWith("/") || /^[A-Za-z]:\//u.test(normalized)) return null;
+	const segments = normalized.split("/");
+	if (segments.some((segment) => segment.length === 0 || segment === "." || segment === "..")) return null;
+	return normalized;
+}
+
+async function hasGitMarker(source: string): Promise<boolean> {
+	let current = resolve(source);
+	while (true) {
+		try {
+			await lstat(resolve(current, ".git"));
+			return true;
+		} catch (error) {
+			const code =
+				typeof error === "object" && error !== null && "code" in error ? (error as { code?: unknown }).code : undefined;
+			if (code !== "ENOENT" && code !== "ENOTDIR") throw error;
+		}
+		const parent = resolve(current, "..");
+		if (parent === current) return false;
+		current = parent;
+	}
+}