@leostera/evalkit 0.9.0 → 0.11.0

package/dist/src-Dc3GXacB.mjs

@@ -0,0 +1,3637 @@
+import { $ as DEFAULT_ARTIFACT_EXCLUDE, B as RunMetadataSchema, Ct as limitsFromParameters, Et as resolveLimits, G as ScoreResultSchema, It as resourceUri, L as RunEventSchema, O as CheckpointResultSchema, Q as canonicalParameters, St as formatLimit, Tt as parseLimits, V as RunScoringSchema, W as RunSummarySchema, Y as none, _ as recordError, _t as contentText, a as assertJudgesIndependent, bt as parseModel, ct as judgePrompt, dt as weightedMean, et as DEFAULT_ATTACHMENT_LIMITS, gt as callsIn, ht as activePath, i as agentRuntimes, it as answerInstruction, kt as relativeDatasetPath, mt as toJsonObject, o as authoringId, pt as toJson, rt as UnreadableAnswer, st as judgeInputNames, ut as readJudgeAnswer, vt as isTreeRecord, xt as toolResultValue, yt as modelName, z as RunManifestSchema } from "./src-D6Ax2t9b.mjs";
+import { a as upgradeManifest, c as normalizeUsage, d as uncachedTokens, f as usageMetrics, l as reportedUsages, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS, u as transcriptUsage } from "./upgrade-rMsGzUEJ.mjs";
+import { n as upgradeScoring, t as convertTrajectory } from "./upgrade-transcript-C8DUykLd.mjs";
+import * as Schema from "effect/Schema";
+import { TranscriptHasher, addIntegrity, parseTranscript, serializeTranscript, validateTranscript } from "@leostera/atxf";
+import { createHash, randomUUID } from "node:crypto";
+import { Data, Effect, Either, Exit, Scope } from "effect";
+import os, { constants, homedir, tmpdir } from "node:os";
+import { appendFile, copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
+import path, { join } from "node:path";
+import { spawn } from "node:child_process";
+import { realpathSync } from "node:fs";
+import { createInterface } from "node:readline";
+import * as ParseResult from "effect/ParseResult";
+//#region ../runner/src/errors.ts
+var AutExecutionError = class extends Data.TaggedError("AutExecutionError") {};
+var DatasetError = class extends Data.TaggedError("DatasetError") {};
+var ReportError = class extends Data.TaggedError("ReportError") {};
+var ScoringError = class extends Data.TaggedError("ScoringError") {};
+var RuleExecutionError = class extends Data.TaggedError("RuleExecutionError") {};
+var CheckpointExecutionError = class extends Data.TaggedError("CheckpointExecutionError") {};
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
+var CancelledError = class extends Data.TaggedError("CancelledError") {};
+function cancelled() {
+	return new CancelledError({ message: "Cancelled" });
+}
+/** The run hit one of its limits; EvalKit stopped it. */
+var LimitReachedError = class extends Data.TaggedError("LimitReachedError") {};
+/** Why `signal` aborted: the limit it hit, or a cancellation. */
+function abortError(signal) {
+	return signal.reason instanceof LimitReachedError ? signal.reason : cancelled();
+}
+/** How long a turn may take to settle after its run hits a limit (RFD0009). */
+const LIMIT_GRACE_MS = 1e4;
+/**
+* Settles like `work`, or rejects once `signal` aborts: at once when
+* cancelled, and after up to `graceMs` when a limit was hit, so an agent
+* that sees `context.signal` can finish what it was writing.
+*/
+async function unlessAborted(work, signal, graceMs = 0) {
+	if (signal.aborted) throw abortError(signal);
+	let onAbort;
+	let timer;
+	const running = work();
+	const aborted = new Promise((_, reject) => {
+		onAbort = () => {
+			const error = abortError(signal);
+			if (!(error instanceof LimitReachedError) || graceMs <= 0) return reject(error);
+			timer = setTimeout(() => reject(error), graceMs);
+		};
+		signal.addEventListener("abort", onAbort, { once: true });
+	});
+	try {
+		return await Promise.race([running, aborted]);
+	} finally {
+		if (timer) clearTimeout(timer);
+		if (onAbort) signal.removeEventListener("abort", onAbort);
+	}
+}
+//#endregion
+//#region ../runner/src/attachments.ts
+const SEGMENT = /^[A-Za-z0-9_-][A-Za-z0-9._-]{0,99}$/;
+const MAX_NAME = 200;
+/**
+* Checks an attachment name: a relative path whose segments are letters,
+* digits, `.`, `_` and `-`, not starting with a dot. That rules out
+* absolute paths, `..`, hidden files, separators other than `/`, and
+* anything a filesystem or URL would read differently.
+*/
+function attachmentName(name) {
+	if (typeof name !== "string" || !name || name.length > MAX_NAME || !name.split("/").every((segment) => SEGMENT.test(segment))) throw new Error(`Attachment name ${JSON.stringify(name)} is not allowed: use a relative path of letters, digits, ".", "_" and "-", like "logs/agent.log"`);
+	return name;
+}
+/** The last `max` bytes, starting on a whole UTF-8 character when the content is text. */
+function lastBytes(bytes, max, text) {
+	let start = bytes.byteLength - max;
+	if (text) while (start < bytes.byteLength && (bytes[start] & 192) === 128) start++;
+	return bytes.subarray(start);
+}
+/**
+* Saves a run's attachments as `attachments/<name>` below its artifacts.
+* Content is stored as given and never interpreted. A file over
+* `maxFileBytes` keeps its end (where a log's errors are); one that would
+* take the run past `maxRunBytes` is rejected.
+*/
+function attachmentStore(writer, limits = {}) {
+	const { maxFileBytes, maxRunBytes } = {
+		...DEFAULT_ATTACHMENT_LIMITS,
+		...limits
+	};
+	const entries = [];
+	const names = /* @__PURE__ */ new Set();
+	const pending = /* @__PURE__ */ new Set();
+	let used = 0;
+	let closed = false;
+	async function attach(name, content) {
+		if (closed) throw new Error(`Cannot attach ${JSON.stringify(name)}: the run has finished`);
+		attachmentName(name);
+		if (typeof content !== "string" && !(content instanceof Uint8Array)) throw new Error(`Attachment ${name} must be a string or a Uint8Array, not ${content === null ? "null" : typeof content}`);
+		if (names.has(name)) throw new Error(`Attachment ${name} was already attached`);
+		const text = typeof content === "string";
+		const original = text ? new TextEncoder().encode(content) : content;
+		const bytes = original.byteLength > maxFileBytes ? lastBytes(original, maxFileBytes, text) : original;
+		if (used + bytes.byteLength > maxRunBytes) throw new Error(`Attachment ${name} (${bytes.byteLength} bytes) would take this run past its ${maxRunBytes}-byte attachment limit (${used} used)`);
+		names.add(name);
+		used += bytes.byteLength;
+		const write = writer.writeArtifact(`attachments/${name}`, bytes);
+		pending.add(write);
+		try {
+			await write;
+			entries.push({
+				name,
+				size: bytes.byteLength,
+				...bytes.byteLength < original.byteLength ? { truncatedFrom: original.byteLength } : {}
+			});
+		} catch (error) {
+			names.delete(name);
+			used -= bytes.byteLength;
+			throw error;
+		} finally {
+			pending.delete(write);
+		}
+	}
+	return {
+		attach,
+		async close() {
+			closed = true;
+			await Promise.allSettled(pending);
+			return [...entries].sort((a, b) => a.name.localeCompare(b.name));
+		}
+	};
+}
+/**
+* An `attach` that remembers what it saved, so a score or checkpoint can
+* link the files its scorer attached, even when the scorer then failed.
+*/
+function trackAttachments(attach) {
+	const attached = [];
+	return {
+		attached,
+		async attach(name, content) {
+			await attach(name, content);
+			attached.push(name);
+		}
+	};
+}
+/** `{ attachments }` when a scorer attached any. */
+const attachedFields = (attached) => attached.length ? { attachments: [...attached] } : {};
+//#endregion
+//#region ../runner/src/run-transcript.ts
+/**
+* One session's ATXF transcript as it is written: every record hashed as
+* it is appended (ATXF §17), saved in order, and sealed at the end. It is
+* the adapter's `TranscriptWriter` and EvalKit's own.
+*/
+var RunTranscript = class {
+	source;
+	header;
+	hasher;
+	entries = [];
+	ids = /* @__PURE__ */ new Set();
+	imported = /* @__PURE__ */ new Set();
+	importBase;
+	leaf = null;
+	next = 0;
+	pendingUser;
+	turns = 0;
+	subagentTranscripts = /* @__PURE__ */ new Map();
+	subagentSeen = /* @__PURE__ */ new Map();
+	sealed = false;
+	writes = Promise.resolve();
+	writeError;
+	constructor(source) {
+		this.source = source;
+		this.hasher = new TranscriptHasher({
+			type: "transcript",
+			atxf: "0.1",
+			id: source.id,
+			time: source.now().toISOString(),
+			source: {
+				agent: source.agent.id,
+				...source.agent.version ? { version: source.agent.version } : {}
+			},
+			fidelity: {
+				topology: "tree",
+				reasoning: "mixed",
+				usage: "per-response",
+				native: "partial",
+				ids: "stable"
+			},
+			ext: { evalkit: {
+				runId: source.runId,
+				scenarioId: source.scenarioId
+			} }
+		});
+		this.header = this.hasher.header;
+		this.save(this.header);
+	}
+	/** The transcript so far. */
+	get transcript() {
+		return {
+			header: this.header,
+			records: [...this.entries]
+		};
+	}
+	/** The ID of the active leaf: where run events are placed. */
+	get leafId() {
+		return this.leaf;
+	}
+	/** Records appended since `from` (a `records.length` taken earlier). */
+	since(from) {
+		return this.entries.slice(from);
+	}
+	get length() {
+		return this.entries.length;
+	}
+	/** Resolves once every line is saved; rejects with the first write error. */
+	async settled() {
+		await this.writes;
+		if (this.writeError) throw this.writeError;
+	}
+	/**
+	* The message EvalKit is about to send. It is written before the adapter's
+	* first record of the turn, unless the adapter writes a user message (or
+	* imports one with the same text) first, which then stands for it.
+	*/
+	scriptUser(message) {
+		this.flushUser();
+		this.pendingUser = message;
+	}
+	/** Writes the scripted message if nothing in the turn stood for it. */
+	endTurn() {
+		this.flushUser();
+	}
+	/** What `send()` returned, as records: tool calls with results, then the reply. */
+	writeTurnResult(result, turn) {
+		if (typeof result === "string") result = { reply: result };
+		if (typeof result !== "object" || result === null) throw new Error("send() must return nothing, a string, or a turn result");
+		const calls = (result.toolCalls ?? []).map((call, index) => ({
+			...call,
+			id: call.id ?? `turn-${turn}-call-${index + 1}`
+		}));
+		const extras = {
+			...result.model === void 0 ? {} : { model: result.model },
+			...result.usage === void 0 ? {} : { usage: result.usage }
+		};
+		const hasReply = result.reply !== void 0;
+		if (calls.length) {
+			this.assistant({
+				content: calls.map((call) => ({
+					type: "tool_call",
+					id: call.id,
+					name: call.name,
+					kind: "function",
+					arguments: toolArguments(call.arguments)
+				})),
+				...hasReply ? {} : extras
+			});
+			for (const call of calls) if (call.result !== void 0) this.toolResult(call.id, call.result, { name: call.name });
+		}
+		if (hasReply) this.assistant({
+			content: typeof result.reply === "string" ? result.reply : JSON.stringify(toJson(result.reply, "reply")),
+			...extras
+		});
+		else if (!calls.length && result.usage) {
+			const usage = normalizeUsage(result.usage);
+			if (usage) this.append({
+				type: "usage",
+				kind: "turn",
+				usage,
+				...result.model === void 0 ? {} : { model: parseModel(result.model) }
+			});
+		}
+	}
+	/** The adapter's writer. */
+	writer() {
+		return {
+			user: (content, options) => {
+				this.pendingUser = void 0;
+				return this.append({
+					type: "message",
+					role: "user",
+					content,
+					...options?.origin ? { origin: options.origin } : {}
+				});
+			},
+			assistant: (message) => this.assistant(message),
+			toolResult: (callId, result, options) => this.toolResult(callId, result, options),
+			append: (record) => this.append(record),
+			import: (transcript) => this.importTranscript(transcript),
+			turn: (phase, options = {}) => {
+				const turnId = options.turnId ?? `turn-${++this.turns}`;
+				this.append({
+					type: "turn",
+					phase,
+					turnId,
+					...options.outcome ? { outcome: options.outcome } : {},
+					...options.error ? { error: options.error } : {}
+				});
+				return turnId;
+			},
+			subagent: (transcript) => this.keepSubagent(transcript)
+		};
+	}
+	/** Sub-agent transcripts kept so far, by transcript ID. */
+	get subagents() {
+		return Object.fromEntries(this.subagentTranscripts);
+	}
+	keepSubagent(transcript) {
+		if (this.sealed) throw new Error("The transcript is sealed: the run has ended");
+		const id = transcript?.header?.id;
+		if (typeof id !== "string" || !id || !Array.isArray(transcript.records)) throw new Error("subagent() takes an ATXF transcript: { header: { id, … }, records: [...] }");
+		if (id === this.header.id) throw new Error(`A sub-agent cannot have the run's transcript ID ${id}`);
+		const seen = this.subagentSeen.get(id) ?? /* @__PURE__ */ new Set();
+		this.subagentSeen.set(id, seen);
+		for (const record of transcript.records) {
+			if (seen.has(record.id)) continue;
+			seen.add(record.id);
+			try {
+				this.source.onSubagentRecord?.(record);
+			} catch {}
+		}
+		this.subagentTranscripts.set(id, transcript);
+		this.saveSubagent(id, transcript);
+	}
+	saveSubagent(id, transcript) {
+		const persist = this.source.persistSubagent;
+		if (!persist) return;
+		const text = serializeTranscript(transcript);
+		this.writes = this.writes.then(() => persist(id, text).catch((error) => {
+			this.writeError ??= error;
+		}));
+	}
+	assistant(message) {
+		const usage = normalizeUsage(message.usage);
+		return this.append({
+			type: "message",
+			role: "assistant",
+			content: typeof message.content === "string" ? [{
+				type: "text",
+				text: message.content
+			}] : message.content,
+			...message.model === void 0 ? {} : { model: parseModel(message.model) },
+			...usage ? { usage } : {},
+			...message.stop ? { stop: message.stop } : {},
+			...message.error ? { error: message.error } : {}
+		});
+	}
+	toolResult(callId, result, options = {}) {
+		const value = toJson(result === void 0 ? null : result, "tool result");
+		return this.append({
+			type: "message",
+			role: "tool",
+			toolCallId: callId,
+			...options.name ? { toolName: options.name } : {},
+			content: typeof value === "string" ? value : JSON.stringify(value),
+			...typeof value === "string" ? {} : { details: { result: value } },
+			...options.isError ? { isError: true } : {}
+		});
+	}
+	/** Appends one tree record after the active leaf (or below `parentId`). */
+	append(input) {
+		if (this.sealed) throw new Error("The transcript is sealed: the run has ended");
+		if (!isTreeRecord(input)) throw new Error(`Transcript records must be ATXF tree records; received type ${JSON.stringify(input.type)}`);
+		this.flushUser();
+		return this.add(input);
+	}
+	add(input) {
+		const id = input.id ?? `ek-${++this.next}`;
+		if (this.ids.has(id)) throw new Error(`Transcript record ID ${id} is already used`);
+		const parentId = input.parentId === void 0 ? this.leaf : input.parentId;
+		if (parentId !== null && !this.ids.has(parentId)) throw new Error(`Transcript record ${id} names parent ${parentId}, which is not earlier in the transcript`);
+		const { hash: _, ...record } = {
+			...input,
+			id,
+			time: input.time ?? this.source.now().toISOString(),
+			parentId
+		};
+		this.push(record);
+		return id;
+	}
+	push(record) {
+		const hashed = this.hasher.push(record);
+		this.entries.push(hashed);
+		this.ids.add(hashed.id);
+		if (isTreeRecord(hashed)) this.leaf = hashed.id;
+		else if (hashed.type === "leaf") this.leaf = hashed.targetId;
+		this.save(hashed);
+		try {
+			this.source.onRecord?.(hashed);
+		} catch {}
+	}
+	flushUser() {
+		if (this.pendingUser === void 0) return;
+		const text = this.pendingUser;
+		this.pendingUser = void 0;
+		this.add({
+			type: "message",
+			role: "user",
+			content: text,
+			origin: {
+				kind: "synthetic",
+				name: "evalkit"
+			}
+		});
+	}
+	importTranscript(transcript) {
+		if (this.sealed) throw new Error("The transcript is sealed: the run has ended");
+		const fresh = transcript.records.filter((record) => record.type !== "seal" && !this.imported.has(record.id));
+		if (!fresh.length) return;
+		if (fresh.findIndex((record) => record.type === "message" && record.role === "user" && contentText(record.content) === this.pendingUser) === -1) this.flushUser();
+		else this.pendingUser = void 0;
+		if (this.importBase === void 0) this.importBase = this.leaf;
+		for (const record of fresh) {
+			if (this.ids.has(record.id)) throw new Error(`Imported record ID ${record.id} is already used in this transcript`);
+			const { hash: _, ...unhashed } = record;
+			const placed = isTreeRecord(record) && record.parentId === null ? {
+				...unhashed,
+				parentId: this.importBase
+			} : unhashed;
+			this.imported.add(record.id);
+			this.push(placed);
+		}
+	}
+	/** Every usage reported so far that ATXF counts toward the total. */
+	*usages() {
+		for (const record of this.entries) if (record.type === "message" && record.role === "assistant") {
+			if (record.usage) yield record.usage;
+		} else if ((record.type === "compaction" || record.type === "branch_summary") && record.usage) yield record.usage;
+		else if (record.type === "usage" && !record.cumulative) yield record.usage;
+	}
+	/** Appends the seal (ATXF §17.2); nothing can be appended after it. */
+	seal() {
+		if (this.sealed) return;
+		this.flushUser();
+		const seal = this.hasher.seal(`ek-seal-${++this.next}`, this.source.now().toISOString());
+		this.entries.push(seal);
+		this.ids.add(seal.id);
+		this.save(seal);
+		this.sealed = true;
+		for (const [id, transcript] of this.subagentTranscripts) {
+			const records = transcript.records.filter((record) => record.type !== "seal");
+			const last = records.at(-1)?.time ?? transcript.header.time;
+			const sealed = addIntegrity({
+				header: transcript.header,
+				records
+			}, { seal: {
+				id: `ek-seal-${id}`,
+				time: last
+			} });
+			this.subagentTranscripts.set(id, sealed);
+			this.saveSubagent(id, sealed);
+		}
+	}
+	/** What the ATXF validator finds wrong with the transcript and its sub-agents'; empty when valid. */
+	problems() {
+		const check = (transcript, prefix) => validateTranscript(serializeTranscript(transcript)).diagnostics.filter((diagnostic) => diagnostic.severity === "error").map((diagnostic) => `${prefix}${diagnostic.line ? `line ${diagnostic.line}: ` : ""}${diagnostic.message}`);
+		return [...check(this.transcript, ""), ...[...this.subagentTranscripts].flatMap(([id, transcript]) => check(transcript, `sub-agent ${id}: `))];
+	}
+	save(line) {
+		const persist = this.source.persist;
+		if (!persist) return;
+		this.writes = this.writes.then(() => persist(line).catch((error) => {
+			this.writeError ??= error;
+		}));
+	}
+};
+/** ATXF tool arguments are an object; anything else is kept under `value`. */
+function toolArguments(value) {
+	if (value === void 0) return {};
+	const json = toJson(value, "tool arguments");
+	return typeof json === "object" && json !== null && !Array.isArray(json) ? json : { value: json };
+}
+//#endregion
+//#region ../runner/src/shell/local-process.ts
+/**
+* Process groups started here and not yet stopped. Each command leads its
+* own group, so the terminal's Ctrl+C no longer reaches it: when this
+* process exits, even at once on a second Ctrl+C, they are killed.
+*/
+const liveGroups = /* @__PURE__ */ new Set();
+let exitHook = false;
+/** Kills `group` when this process exits, until `forgetGroup`. */
+function rememberGroup(group) {
+	liveGroups.add(group);
+	if (exitHook) return;
+	exitHook = true;
+	process.once("exit", () => {
+		for (const pid of liveGroups) try {
+			process.kill(-pid, "SIGKILL");
+		} catch {}
+	});
+}
+function forgetGroup(group) {
+	liveGroups.delete(group);
+}
+/** Output kept per stream: the last megabyte, where errors are. */
+const OUTPUT_LIMIT = 1e6;
+/** How long a process gets to stop after SIGTERM before SIGKILL. */
+const STOP_GRACE_MS = 2e3;
+/** A process's text output, keeping the last `OUTPUT_LIMIT` characters. */
+var OutputBuffer = class {
+	#text = "";
+	append(chunk) {
+		this.#text += chunk;
+		if (this.#text.length > OUTPUT_LIMIT * 2) this.#text = this.#text.slice(-1e6);
+	}
+	toString() {
+		return this.#text.length > OUTPUT_LIMIT ? this.#text.slice(-1e6) : this.#text;
+	}
+};
+/** The exit code a shell reports for a signal: 128 plus its number. */
+function signalExitCode(signal) {
+	return 128 + (constants.signals[signal] ?? 15);
+}
+/**
+* Starts a process on this machine, in its own process group, and reads its
+* output as it comes. A command that cannot start exits 127 with the reason
+* on stderr, as a shell would.
+*/
+function startLocalProcess(argv, options) {
+	const [program, ...args] = argv;
+	if (!program) throw new Error("A command needs at least a program to run");
+	const stdout = new OutputBuffer();
+	const stderr = new OutputBuffer();
+	const listeners = /* @__PURE__ */ new Set();
+	const emit = (chunk, stream) => {
+		(stream === "stdout" ? stdout : stderr).append(chunk);
+		for (const listener of listeners) listener(chunk, stream);
+	};
+	let child;
+	let timedOut = false;
+	let exited = false;
+	let resolveExit;
+	const exit = new Promise((resolve) => {
+		resolveExit = resolve;
+	});
+	const finish = (result) => {
+		if (exited) return;
+		exited = true;
+		cleanup();
+		resolveExit(timedOut ? {
+			...result,
+			exitCode: 124,
+			timedOut: true
+		} : result);
+	};
+	const signalGroup = async (signal) => {
+		if (exited || child?.pid === void 0) return;
+		try {
+			process.kill(-child.pid, signal);
+		} catch {
+			try {
+				child.kill(signal);
+			} catch {}
+		}
+	};
+	const kill = options.kill ?? signalGroup;
+	/** SIGTERM, then SIGKILL if it is still running after the grace. */
+	const stop = async () => {
+		await kill("SIGTERM");
+		const timer = setTimeout(() => void kill("SIGKILL"), STOP_GRACE_MS);
+		timer.unref?.();
+		await exit;
+		clearTimeout(timer);
+	};
+	const onAbort = () => void stop();
+	let timer;
+	const cleanup = () => {
+		options.signal?.removeEventListener("abort", onAbort);
+		if (timer) clearTimeout(timer);
+	};
+	try {
+		child = spawn(program, args, {
+			cwd: options.cwd,
+			env: {
+				...options.env,
+				PWD: options.cwd
+			},
+			detached: true,
+			stdio: [
+				"pipe",
+				"pipe",
+				"pipe"
+			]
+		});
+	} catch (error) {
+		emit(`${errorMessage(error)}\n`, "stderr");
+		finish({ exitCode: 127 });
+	}
+	if (child) {
+		const group = child.pid;
+		if (group !== void 0) {
+			rememberGroup(group);
+			if (options.forgetOnExit !== false) exit.then(() => forgetGroup(group));
+		}
+		child.stdout?.setEncoding("utf8").on("data", (chunk) => emit(chunk, "stdout"));
+		child.stderr?.setEncoding("utf8").on("data", (chunk) => emit(chunk, "stderr"));
+		child.on("error", (error) => {
+			emit(`${program}: ${errorMessage(error)}\n`, "stderr");
+			finish({ exitCode: 127 });
+		});
+		child.on("close", (code, signal) => finish(signal ? {
+			exitCode: signalExitCode(signal),
+			signal
+		} : { exitCode: code ?? 0 }));
+		child.stdin?.on("error", () => {});
+		child.stdin?.end(options.input ?? "");
+		if (options.signal?.aborted) onAbort();
+		else options.signal?.addEventListener("abort", onAbort, { once: true });
+		if (options.timeoutMs !== void 0) {
+			timer = setTimeout(() => {
+				timedOut = true;
+				stop();
+			}, options.timeoutMs);
+			timer.unref?.();
+		}
+	}
+	return {
+		get pid() {
+			return child?.pid;
+		},
+		stdout: () => stdout.toString(),
+		stderr: () => stderr.toString(),
+		onOutput(listener) {
+			listeners.add(listener);
+			return () => listeners.delete(listener);
+		},
+		wait: () => exit,
+		async kill(signal = "SIGTERM") {
+			await kill(signal);
+		},
+		killLocal: signalGroup
+	};
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+/** Runs a process to its end and returns its output. */
+async function collect(process) {
+	return {
+		...await process.wait(),
+		stdout: process.stdout(),
+		stderr: process.stderr()
+	};
+}
+/**
+* The processes a run's shell started, so they stop when the run ends:
+* SIGTERM, then SIGKILL after a grace.
+*/
+var ProcessTracker = class {
+	#live = /* @__PURE__ */ new Set();
+	#closed = false;
+	track(process) {
+		if (this.#closed) {
+			process.kill("SIGKILL");
+			throw new Error("The run is over: its shell takes no more commands");
+		}
+		this.#live.add(process);
+		process.wait().then(() => this.#live.delete(process));
+		return process;
+	}
+	get closed() {
+		return this.#closed;
+	}
+	async close() {
+		this.#closed = true;
+		await Promise.all([...this.#live].map(async (process) => {
+			await process.kill("SIGTERM");
+			const timer = setTimeout(() => void process.kill("SIGKILL"), STOP_GRACE_MS);
+			timer.unref?.();
+			await process.wait();
+			clearTimeout(timer);
+		}));
+	}
+};
+//#endregion
+//#region ../runner/src/shell/docker.ts
+/** Runs the container runtime's CLI and returns its trimmed stdout; throws with its stderr. */
+async function docker(runtime, args, options = {}) {
+	const result = await collect(startLocalProcess([runtime, ...args], {
+		cwd: process.cwd(),
+		env: process.env,
+		...options.input !== void 0 ? { input: options.input } : {},
+		...options.signal ? { signal: options.signal } : {}
+	}));
+	if (result.exitCode !== 0) throw new Error(`${runtime} ${args[0]} failed (exit ${result.exitCode}): ${result.stderr.trim() || result.stdout.trim()}`);
+	return result.stdout.trim();
+}
+const runtimes = /* @__PURE__ */ new Map();
+/** Checks once per process that the runtime's CLI and its daemon answer. */
+function ensureRuntime(runtime) {
+	let checked = runtimes.get(runtime);
+	if (!checked) {
+		checked = docker(runtime, [
+			"version",
+			"--format",
+			"{{.Server.Version}}"
+		]).then(() => void 0).catch((error) => {
+			runtimes.delete(runtime);
+			throw new Error(`Container isolation needs the ${runtime} CLI and a running engine, which did not answer: ${error instanceof Error ? error.message : String(error)}. Install and start Docker Desktop, OrbStack, Colima, Rancher Desktop or podman, or name another docker-compatible command with container({ runtime }).`);
+		});
+		runtimes.set(runtime, checked);
+	}
+	return checked;
+}
+/** True for an image that names a Dockerfile (or its directory) rather than an image reference. */
+function isDockerfile(image) {
+	return image.startsWith("./") || image.startsWith("../") || image.startsWith("/") || /(^|\/)Dockerfile[^/]*$/i.test(image);
+}
+const IGNORED = /* @__PURE__ */ new Set([
+	"node_modules",
+	".git",
+	"_evalkit-results"
+]);
+/** A hash of a build context: each file's path, size and modification time. */
+async function contextHash(directory, dockerfile) {
+	const hash = createHash("sha256");
+	hash.update(await readFile(dockerfile));
+	const walk = async (dir) => {
+		const entries = await readdir(dir, { withFileTypes: true });
+		entries.sort((a, b) => a.name.localeCompare(b.name));
+		for (const entry of entries) {
+			if (IGNORED.has(entry.name)) continue;
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) await walk(full);
+			else if (entry.isFile()) {
+				const info = await stat(full);
+				hash.update(`${path.relative(directory, full)}\0${info.size}\0${info.mtimeMs}\n`);
+			}
+		}
+	};
+	await walk(directory);
+	return hash.digest("hex").slice(0, 16);
+}
+const builds = /* @__PURE__ */ new Map();
+/**
+* The image a run's container starts from: a reference as it is, or a
+* Dockerfile built once per process and tagged by the hash of its
+* directory, reused while the directory is unchanged.
+*/
+function resolveImage(runtime, image, projectRoot) {
+	if (!isDockerfile(image)) return Promise.resolve(image);
+	const target = path.resolve(projectRoot, image);
+	const key = `${runtime}\0${target}`;
+	let built = builds.get(key);
+	if (!built) {
+		built = (async () => {
+			const info = await stat(target).catch(() => void 0);
+			if (!info) throw new Error(`container() image ${image} is not an image name or a Dockerfile in the project: ${target} does not exist`);
+			const dockerfile = info.isDirectory() ? path.join(target, "Dockerfile") : target;
+			const context = path.dirname(dockerfile);
+			const tag = `evalkit-${path.basename(context).toLowerCase().replace(/[^a-z0-9_.-]/g, "-").replace(/^[^a-z0-9]+/, "") || "image"}:${await contextHash(context, dockerfile)}`;
+			if (!await docker(runtime, [
+				"image",
+				"inspect",
+				tag
+			]).then(() => true, () => false)) await docker(runtime, [
+				"build",
+				"-q",
+				"-t",
+				tag,
+				"-f",
+				dockerfile,
+				context
+			]);
+			return tag;
+		})();
+		builds.set(key, built);
+		built.catch(() => builds.delete(key));
+	}
+	return built;
+}
+//#endregion
+//#region ../runner/src/shell/workspace-files.ts
+/**
+* A path relative to the workspace, as an absolute host path; one that
+* would leave the workspace is an error.
+*/
+function insideWorkspace(root, relative = ".") {
+	if (path.isAbsolute(relative)) throw new Error(`Shell paths are relative to the workspace; received ${relative}`);
+	return inside(root, relative, relative);
+}
+function inside(root, file, shown) {
+	const resolved = path.resolve(root, file);
+	const back = path.relative(root, resolved);
+	if (back.startsWith("..") || path.isAbsolute(back)) throw new Error(`${shown} is outside the run's workspace and home`);
+	return resolved;
+}
+/**
+* A shell path as a host path: relative ones are in the workspace,
+* absolute ones must be inside the workspace or the home as commands see
+* them.
+*/
+function hostPath(directories, file = ".") {
+	if (!path.isAbsolute(file)) return inside(directories.root, file, file);
+	for (const [seen, host] of [[directories.seenHome, directories.home], [directories.seenRoot, directories.root]]) {
+		const back = path.relative(seen, file);
+		if (!back.startsWith("..") && !path.isAbsolute(back)) return inside(host, back, file);
+	}
+	throw new Error(`${file} is outside the run's workspace and home`);
+}
+/**
+* The shell's file operations, on the host's copies of the workspace and
+* home: for container isolation they are bind-mounted, so the container
+* sees the same files.
+*/
+function shellFiles(directories) {
+	const resolve = (file) => hostPath(directories, file);
+	return {
+		home: directories.seenHome,
+		readFile: async (file) => readFile(resolve(file), "utf8"),
+		async writeFile(file, content) {
+			const target = resolve(file);
+			await mkdir(path.dirname(target), { recursive: true });
+			await writeFile(target, content);
+		},
+		async list(directory = ".") {
+			return (await readdir(resolve(directory), { withFileTypes: true })).map((entry) => entry.isDirectory() ? `${entry.name}/` : entry.name).sort();
+		}
+	};
+}
+//#endregion
+//#region ../runner/src/shell/container-shell.ts
+/** Where the candidate workspace is inside the container. */
+const CONTAINER_WORKSPACE = "/workspace";
+/** Where the run's private home is mounted. */
+const CONTAINER_HOME = "/evalkit-home";
+/** Where each command records its process ID, so `kill` can reach it. */
+const PID_DIR = "/tmp/.evalkit";
+/** The container's name for a run. */
+function containerName(runId) {
+	return `evalkit-${runId.replace(/[^a-zA-Z0-9_.-]/g, "-")}`;
+}
+/** Removes a run's kept container, if there is one; true when it removed one. */
+async function removeRunContainer(runId, runtime = "docker") {
+	return docker(runtime, [
+		"rm",
+		"--force",
+		containerName(runId)
+	]).then(() => true, () => false);
+}
+/** Arguments for `docker run`: everything but the image and its command. */
+function containerRunArgs(options) {
+	const { isolation } = options;
+	const network = isolation.network === "none" ? ["--network", "none"] : isolation.network === "fixtures" ? ["--network", options.fixturesNetwork ?? "none"] : ["--add-host", "host.docker.internal:host-gateway"];
+	const user = process.platform === "linux" && process.getuid && process.getgid ? [
+		"--user",
+		`${process.getuid()}:${process.getgid()}`,
+		"-e",
+		"HOME=/tmp"
+	] : [];
+	return [
+		"run",
+		"--detach",
+		"--init",
+		"--name",
+		options.name,
+		"--label",
+		`evalkit.run=${options.runId}`,
+		"--volume",
+		`${options.workspace}:${CONTAINER_WORKSPACE}`,
+		"--volume",
+		`${options.home}:${CONTAINER_HOME}`,
+		"--workdir",
+		CONTAINER_WORKSPACE,
+		...network,
+		...user,
+		...isolation.env.flatMap((name) => ["--env", name]),
+		...isolation.mounts.flatMap((mount) => ["--volume", `${path.resolve(mount.host)}:${mount.path}:ro`]),
+		"--entrypoint",
+		"sh"
+	];
+}
+/**
+* The run's container, started before the session: its own process table
+* and filesystem, the candidate workspace bind-mounted read-write, the
+* evaluator workspace never mounted. Commands run in it with `exec`.
+*/
+async function containerShell(options) {
+	const { isolation } = options;
+	const runtime = isolation.runtime;
+	await ensureRuntime(runtime);
+	const image = await resolveImage(runtime, isolation.image, options.projectRoot);
+	const name = containerName(options.runId);
+	const workspace = await realpath(options.root);
+	await docker(runtime, [
+		...containerRunArgs({
+			isolation,
+			name,
+			runId: options.runId,
+			workspace,
+			home: await realpath(options.home),
+			...options.fixturesNetwork ? { fixturesNetwork: options.fixturesNetwork } : {}
+		}),
+		image,
+		"-c",
+		`mkdir -p ${PID_DIR}; trap 'exit 0' TERM; while :; do sleep 3600 & wait $!; done`
+	]);
+	const tracker = new ProcessTracker();
+	const clients = /* @__PURE__ */ new Set();
+	let commands = 0;
+	const start = (command, commandOptions = {}) => {
+		const id = ++commands;
+		const pidFile = `${PID_DIR}/${id}.pid`;
+		const program = typeof command === "string" ? [
+			"sh",
+			"-c",
+			command
+		] : [...command];
+		if (!program.length) throw new Error("A command needs a program to run");
+		const cwd = path.posix.join(CONTAINER_WORKSPACE, commandOptions.cwd ?? ".");
+		if (cwd !== "/workspace" && !cwd.startsWith(`/workspace/`)) throw new Error(`${commandOptions.cwd} is outside the run's workspace`);
+		const signalInside = async (signal) => {
+			const name_ = signal.replace(/^SIG/, "");
+			await docker(runtime, [
+				"exec",
+				name,
+				"sh",
+				"-c",
+				`p=$(cat ${pidFile} 2>/dev/null) && { kill -s ${name_} -- -$p 2>/dev/null || kill -s ${name_} $p; }`
+			]).catch(() => void 0);
+		};
+		const started = startLocalProcess([
+			runtime,
+			"exec",
+			"--interactive",
+			"--workdir",
+			cwd,
+			...Object.entries(commandOptions.env ?? {}).flatMap(([key, value]) => ["--env", `${key}=${value}`]),
+			name,
+			"sh",
+			"-c",
+			`echo $$ > ${pidFile}; exec "$@"`,
+			"sh",
+			...program
+		], {
+			cwd: process.cwd(),
+			env: process.env,
+			signal: commandOptions.signal ?? options.signal,
+			...commandOptions.timeoutMs !== void 0 ? { timeoutMs: commandOptions.timeoutMs } : {},
+			...commandOptions.input !== void 0 ? { input: commandOptions.input } : {},
+			kill: signalInside
+		});
+		clients.add(started);
+		started.wait().then(() => clients.delete(started));
+		let pid;
+		(async () => {
+			for (let attempt = 0; attempt < 20 && pid === void 0; attempt++) {
+				const text = await docker(runtime, [
+					"exec",
+					name,
+					"cat",
+					pidFile
+				]).catch(() => "");
+				const parsed = Number.parseInt(text, 10);
+				if (Number.isInteger(parsed)) pid = parsed;
+				else await new Promise((resolve) => setTimeout(resolve, 50));
+			}
+		})();
+		const process_ = {
+			get pid() {
+				return pid;
+			},
+			stdout: started.stdout,
+			stderr: started.stderr,
+			onOutput: started.onOutput,
+			wait: started.wait,
+			kill: started.kill
+		};
+		return tracker.track(process_);
+	};
+	return {
+		isolation: "container",
+		root: CONTAINER_WORKSPACE,
+		container: name,
+		exec: async (command, commandOptions) => collect(start(command, commandOptions)),
+		spawn: (command, commandOptions) => start(command, commandOptions),
+		...shellFiles({
+			root: options.root,
+			seenRoot: CONTAINER_WORKSPACE,
+			home: options.home,
+			seenHome: CONTAINER_HOME
+		}),
+		async close({ keep = false } = {}) {
+			await docker(runtime, keep ? [
+				"stop",
+				"--time",
+				"2",
+				name
+			] : [
+				"rm",
+				"--force",
+				name
+			]).catch(() => void 0);
+			for (const client of clients) await client.killLocal("SIGKILL");
+			await tracker.close();
+		}
+	};
+}
+//#endregion
+//#region ../runner/src/shell/host-shell.ts
+/** A string runs with `sh -c`; an array runs as it is. */
+function argv(command) {
+	if (typeof command === "string") return [
+		"/bin/sh",
+		"-c",
+		command
+	];
+	if (!command.length) throw new Error("A command needs a program to run");
+	return [...command];
+}
+/**
+* Commands on this machine, in the run's workspace, with the host's
+* environment: what adapters did before EvalKit had a shell. Each command
+* leads its own process group, and every group is stopped when the run
+* ends, background children included.
+*/
+function hostShell(options) {
+	const tracker = new ProcessTracker();
+	const groups = /* @__PURE__ */ new Set();
+	const start = (command, commandOptions = {}) => {
+		const process_ = startLocalProcess(argv(command), {
+			cwd: insideWorkspace(options.root, commandOptions.cwd),
+			env: {
+				...process.env,
+				...commandOptions.env
+			},
+			signal: commandOptions.signal ?? options.signal,
+			...commandOptions.timeoutMs !== void 0 ? { timeoutMs: commandOptions.timeoutMs } : {},
+			...commandOptions.input !== void 0 ? { input: commandOptions.input } : {},
+			forgetOnExit: false
+		});
+		if (process_.pid !== void 0) groups.add(process_.pid);
+		return tracker.track(process_);
+	};
+	return {
+		isolation: "none",
+		root: options.root,
+		exec: async (command, commandOptions) => collect(start(command, commandOptions)),
+		spawn: (command, commandOptions) => start(command, commandOptions),
+		...shellFiles({
+			root: options.root,
+			seenRoot: options.root,
+			home: options.home,
+			seenHome: options.home
+		}),
+		async close() {
+			await tracker.close();
+			for (const group of groups) {
+				try {
+					process.kill(-group, "SIGKILL");
+				} catch {}
+				forgetGroup(group);
+			}
+		}
+	};
+}
+//#endregion
+//#region ../runner/src/shell/sandbox-shell.ts
+/**
+* Runs inside the sandbox and starts every command of the run as its
+* child, so all of them share one sandbox: a later command may signal a
+* server an earlier one started, and nothing outside. Plain JavaScript, so
+* Node and Bun both run it. One JSON message per line each way.
+*/
+const SUPERVISOR = `import { spawn } from 'node:child_process';
+import { createInterface } from 'node:readline';
+const children = new Map();
+const send = (message) => process.stdout.write(JSON.stringify(message) + '\\n');
+const signal = (child, name) => {
+  try { process.kill(-child.pid, name); } catch { try { child.kill(name); } catch {} }
+};
+createInterface({ input: process.stdin }).on('line', (line) => {
+  const message = JSON.parse(line);
+  if (message.op === 'kill') {
+    const child = children.get(message.id);
+    if (child) signal(child, message.signal);
+    return;
+  }
+  let child;
+  try {
+    child = spawn(message.argv[0], message.argv.slice(1), {
+      cwd: message.cwd, env: { ...message.env, PWD: message.cwd }, detached: true, stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  } catch (error) {
+    send({ id: message.id, stream: 'stderr', data: String(error.message) + '\\n' });
+    send({ id: message.id, exit: 127 });
+    return;
+  }
+  children.set(message.id, child);
+  send({ id: message.id, pid: child.pid });
+  child.stdout.setEncoding('utf8').on('data', (data) => send({ id: message.id, stream: 'stdout', data }));
+  child.stderr.setEncoding('utf8').on('data', (data) => send({ id: message.id, stream: 'stderr', data }));
+  child.on('error', (error) => {
+    send({ id: message.id, stream: 'stderr', data: message.argv[0] + ': ' + error.message + '\\n' });
+    send({ id: message.id, exit: 127 });
+  });
+  child.on('close', (code, name) => {
+    children.delete(message.id);
+    send({ id: message.id, exit: code, signal: name });
+  });
+  child.stdin.on('error', () => {});
+  child.stdin.end(message.input ?? '');
+});
+process.stdin.on('end', () => {
+  for (const child of children.values()) signal(child, 'SIGKILL');
+  process.exit(0);
+});
+`;
+const real = (file) => {
+	try {
+		return realpathSync(file);
+	} catch {
+		return path.resolve(file);
+	}
+};
+const quote = (file) => JSON.stringify(real(file));
+/**
+* The `sandbox-exec` profile for one run. Everything is allowed but
+* reading or writing the project, the home directory and the other runs'
+* workspaces, and signalling processes outside the sandbox; the run's own
+* workspace and `allow` stay usable. Later rules win.
+*/
+function sandboxProfile(options) {
+	const runs = path.dirname(path.dirname(options.workspace));
+	const denied = [
+		options.projectRoot,
+		homedir(),
+		runs,
+		...options.deny ?? []
+	].map((file) => `(subpath ${quote(file)})`).join(" ");
+	const usable = [options.workspace, ...options.allow].map((file) => `(subpath ${quote(file)})`).join(" ");
+	const readable = [...options.readable.map((file) => `(literal ${quote(file)})`), ...(options.read ?? []).map((file) => `(subpath ${quote(file)})`)].join(" ");
+	return [
+		"(version 1)",
+		"(allow default)",
+		`(deny file-read* file-write* ${denied})`,
+		`(allow file-read* file-write* ${usable})`,
+		...readable ? [`(allow file-read* ${readable})`] : [],
+		"(allow file-read-metadata)",
+		"(deny signal)",
+		"(allow signal (target same-sandbox))"
+	].join("\n");
+}
+/** Variables every run gets when the environment is scrubbed: what programs expect to find. */
+const ESSENTIAL_ENV = [
+	"PATH",
+	"HOME",
+	"TMPDIR",
+	"USER",
+	"LANG",
+	"TERM"
+];
+/** The environment a sandboxed run starts from: the host's, or only the named variables. */
+function sandboxEnv(names, host = process.env) {
+	if (!names) return { ...host };
+	const env = {};
+	for (const name of [...ESSENTIAL_ENV, ...names]) if (host[name] !== void 0) env[name] = host[name];
+	return env;
+}
+const resolveFrom = (root) => (file) => path.resolve(root, file.startsWith("~/") ? path.join(homedir(), file.slice(2)) : file);
+/** Writes the supervisor script once, where the sandbox can read it. */
+async function supervisorScript() {
+	const hash = createHash("sha256").update(SUPERVISOR).digest("hex").slice(0, 12);
+	const file = path.join(tmpdir(), `evalkit-supervisor-${hash}.mjs`);
+	await writeFile(file, SUPERVISOR);
+	return file;
+}
+/**
+* macOS `sandbox-exec` isolation: one sandboxed supervisor per run, which
+* starts every command the run sends it.
+*/
+async function sandboxShell(options) {
+	if (process.platform !== "darwin") throw new Error("sandboxExec() isolation needs macOS sandbox-exec; use container() on other systems");
+	const script = await supervisorScript();
+	const baseEnv = sandboxEnv(options.isolation.env);
+	const profile = sandboxProfile({
+		workspace: real(options.root),
+		projectRoot: real(options.projectRoot),
+		allow: [real(options.home), ...options.isolation.allow.map(resolveFrom(options.projectRoot))],
+		read: options.isolation.read.map(resolveFrom(options.projectRoot)),
+		deny: options.isolation.deny.map(resolveFrom(options.projectRoot)),
+		readable: [process.execPath, script]
+	});
+	const supervisor = spawn("sandbox-exec", [
+		"-p",
+		profile,
+		process.execPath,
+		script
+	], {
+		cwd: options.root,
+		env: baseEnv,
+		stdio: [
+			"pipe",
+			"pipe",
+			"pipe"
+		]
+	});
+	let supervisorError = "";
+	supervisor.stderr.setEncoding("utf8").on("data", (chunk) => {
+		supervisorError = (supervisorError + chunk).slice(-2e3);
+	});
+	const handlers = /* @__PURE__ */ new Map();
+	createInterface({ input: supervisor.stdout }).on("line", (line) => {
+		let message;
+		try {
+			message = JSON.parse(line);
+		} catch {
+			return;
+		}
+		handlers.get(message.id)?.(message);
+	});
+	const gone = new Promise((resolve) => supervisor.on("close", () => resolve()));
+	gone.then(() => {
+		for (const handler of handlers.values()) handler({
+			id: -1,
+			stream: "stderr",
+			data: `The run's sandbox stopped: ${supervisorError.trim()}\n`
+		});
+		for (const [id, handler] of handlers) handler({
+			id,
+			exit: 126
+		});
+	});
+	const send = (message) => {
+		if (supervisor.stdin.writable) supervisor.stdin.write(`${JSON.stringify(message)}\n`);
+	};
+	const tracker = new ProcessTracker();
+	let commands = 0;
+	const start = (command, commandOptions = {}) => {
+		const id = ++commands;
+		const stdout = new OutputBuffer();
+		const stderr = new OutputBuffer();
+		const listeners = /* @__PURE__ */ new Set();
+		let pid;
+		let timedOut = false;
+		let resolveExit;
+		const exit = new Promise((resolve) => {
+			resolveExit = resolve;
+		});
+		let exited = false;
+		const kill = async (signal) => {
+			if (!exited) send({
+				op: "kill",
+				id,
+				signal
+			});
+		};
+		const stop = async () => {
+			await kill("SIGTERM");
+			const timer = setTimeout(() => void kill("SIGKILL"), STOP_GRACE_MS);
+			timer.unref?.();
+			await exit;
+			clearTimeout(timer);
+		};
+		const onAbort = () => void stop();
+		const signal = commandOptions.signal ?? options.signal;
+		const timer = commandOptions.timeoutMs === void 0 ? void 0 : setTimeout(() => {
+			timedOut = true;
+			stop();
+		}, commandOptions.timeoutMs);
+		timer?.unref?.();
+		handlers.set(id, (message) => {
+			if ("pid" in message) pid = message.pid;
+			else if ("stream" in message) {
+				(message.stream === "stdout" ? stdout : stderr).append(message.data);
+				for (const listener of listeners) listener(message.data, message.stream);
+			} else if ("exit" in message && !exited) {
+				exited = true;
+				handlers.delete(id);
+				signal.removeEventListener("abort", onAbort);
+				if (timer) clearTimeout(timer);
+				const result = message.signal ? {
+					exitCode: signalExitCode(message.signal),
+					signal: message.signal
+				} : { exitCode: message.exit ?? 0 };
+				resolveExit(timedOut ? {
+					...result,
+					exitCode: 124,
+					timedOut: true
+				} : result);
+			}
+		});
+		send({
+			op: "spawn",
+			id,
+			argv: argv(command),
+			cwd: insideWorkspace(options.root, commandOptions.cwd),
+			env: {
+				...baseEnv,
+				...commandOptions.env
+			},
+			...commandOptions.input !== void 0 ? { input: commandOptions.input } : {}
+		});
+		if (signal.aborted) onAbort();
+		else signal.addEventListener("abort", onAbort, { once: true });
+		return tracker.track({
+			get pid() {
+				return pid;
+			},
+			stdout: () => stdout.toString(),
+			stderr: () => stderr.toString(),
+			onOutput(listener) {
+				listeners.add(listener);
+				return () => listeners.delete(listener);
+			},
+			wait: () => exit,
+			kill: async (name = "SIGTERM") => kill(name)
+		});
+	};
+	return {
+		isolation: "sandbox-exec",
+		root: options.root,
+		exec: async (command, commandOptions) => {
+			const started = start(command, commandOptions);
+			return {
+				...await started.wait(),
+				stdout: started.stdout(),
+				stderr: started.stderr()
+			};
+		},
+		spawn: (command, commandOptions) => start(command, commandOptions),
+		...shellFiles({
+			root: options.root,
+			seenRoot: options.root,
+			home: options.home,
+			seenHome: options.home
+		}),
+		async close() {
+			await tracker.close();
+			supervisor.stdin.end();
+			const timer = setTimeout(() => supervisor.kill("SIGKILL"), STOP_GRACE_MS);
+			timer.unref?.();
+			await gone;
+			clearTimeout(timer);
+		}
+	};
+}
+//#endregion
+//#region ../runner/src/shell/fixtures.ts
+/** How long a fixture has to answer `ready`. */
+const FIXTURE_READY_MS = 6e4;
+const POLL_MS = 250;
+/** Forwards a port to the host; busybox's nc serves each connection in its own process. */
+const PROXY_IMAGE = "busybox:stable";
+const NAME = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
+/** The port a fixture listens on: its own, or its `ready` URL's. */
+function fixturePort(fixture) {
+	if (fixture.port !== void 0) return fixture.port;
+	const url = readyUrl(fixture.ready);
+	if (!url) return void 0;
+	if (url.port) return Number(url.port);
+	return url.protocol === "https:" ? 443 : 80;
+}
+function readyUrl(ready) {
+	if (!ready || !/^https?:\/\//.test(ready)) return void 0;
+	try {
+		return new URL(ready);
+	} catch {
+		return;
+	}
+}
+/** Checks fixture definitions from the config. */
+function parseFixtures(fixtures) {
+	for (const [name, fixture] of Object.entries(fixtures)) {
+		if (!NAME.test(name)) throw new Error(`Fixture names are lowercase letters, digits and dashes, so runs can reach them as fixtures.<name>; received ${JSON.stringify(name)}`);
+		if (typeof fixture?.start !== "string" || !fixture.start.trim()) throw new Error(`Fixture ${name} needs a start command`);
+		if (fixture.port !== void 0 && (!Number.isInteger(fixture.port) || fixture.port < 1 || fixture.port > 65535)) throw new Error(`Fixture ${name} port must be from 1 to 65535`);
+	}
+	return { ...fixtures };
+}
+/**
+* The project's fixtures for one invocation: started on the host, in the
+* project root, the first time a run needs them, and stopped after its last.
+*/
+function fixtureSet(fixtures, options) {
+	const running = Object.entries(parseFixtures(fixtures)).map(([name, fixture]) => ({
+		name,
+		fixture,
+		port: fixturePort(fixture),
+		output: new OutputBuffer()
+	}));
+	const id = randomUUID().slice(0, 8);
+	let starting;
+	let networking;
+	const proxies = [];
+	const shell = (command, output) => {
+		const started = startLocalProcess([
+			"/bin/sh",
+			"-c",
+			command
+		], {
+			cwd: options.root,
+			env: process.env
+		});
+		if (output) started.onOutput((chunk) => output.append(chunk));
+		return started;
+	};
+	const waitReady = async (entry) => {
+		const { ready } = entry.fixture;
+		if (!ready) return;
+		const url = readyUrl(ready);
+		const deadline = Date.now() + FIXTURE_READY_MS;
+		let exited = false;
+		entry.process?.wait().then(() => {
+			exited = true;
+		});
+		while (Date.now() < deadline) {
+			if (exited) throw new Error(`Fixture ${entry.name} exited before it was ready: ${entry.output.toString().trim().slice(-500)}`);
+			if (url ? await fetch(url, { signal: AbortSignal.timeout(1e3) }).then(() => true, () => false) : (await collect(shell(ready))).exitCode === 0) return;
+			await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+		}
+		throw new Error(`Fixture ${entry.name} was not ready after ${FIXTURE_READY_MS / 1e3}s (${ready}): ${entry.output.toString().trim().slice(-500)}`);
+	};
+	return {
+		get started() {
+			return starting !== void 0;
+		},
+		ensure() {
+			starting ??= (async () => {
+				for (const entry of running) entry.process = shell(entry.fixture.start, entry.output);
+				await Promise.all(running.map(waitReady));
+			})();
+			return starting;
+		},
+		addresses(isolation) {
+			const addresses = {};
+			for (const entry of running) {
+				const scheme = readyUrl(entry.fixture.ready)?.protocol ?? "http:";
+				if (isolation.kind !== "container") {
+					if (entry.port !== void 0) addresses[entry.name] = { url: `${scheme}//localhost:${entry.port}` };
+					continue;
+				}
+				if (entry.port === void 0 || isolation.network === "none") continue;
+				addresses[entry.name] = { url: isolation.network === "fixtures" ? `${scheme}//fixtures.${entry.name}:${entry.port}` : `${scheme}//host.docker.internal:${entry.port}` };
+			}
+			return addresses;
+		},
+		network(runtime) {
+			if (networking && networking.runtime !== runtime) throw new Error(`Fixtures are on ${networking.runtime}'s network; one invocation uses one container runtime`);
+			networking ??= {
+				runtime,
+				network: (async () => {
+					await ensureRuntime(runtime);
+					const network = `evalkit-fixtures-${id}`;
+					await docker(runtime, [
+						"network",
+						"create",
+						"--internal",
+						"--label",
+						"evalkit.fixtures=true",
+						network
+					]);
+					for (const entry of running) {
+						if (entry.port === void 0) continue;
+						const proxy = `${network}-${entry.name}`;
+						await docker(runtime, [
+							"run",
+							"--detach",
+							"--rm",
+							"--name",
+							proxy,
+							"--label",
+							"evalkit.fixtures=true",
+							"--add-host",
+							"host.docker.internal:host-gateway",
+							PROXY_IMAGE,
+							"nc",
+							"-lk",
+							"-p",
+							String(entry.port),
+							"-e",
+							"nc",
+							"host.docker.internal",
+							String(entry.port)
+						]);
+						proxies.push({
+							runtime,
+							name: proxy
+						});
+						await docker(runtime, [
+							"network",
+							"connect",
+							"--alias",
+							`fixtures.${entry.name}`,
+							network,
+							proxy
+						]);
+					}
+					return network;
+				})()
+			};
+			return networking.network;
+		},
+		logs: () => running.filter((entry) => entry.process).map((entry) => ({
+			name: entry.name,
+			content: entry.output.toString()
+		})),
+		async stop() {
+			for (const proxy of proxies.splice(0)) await docker(proxy.runtime, [
+				"rm",
+				"--force",
+				proxy.name
+			]).catch(() => void 0);
+			if (networking) {
+				const { runtime, network } = networking;
+				networking = void 0;
+				const name = await network.catch(() => void 0);
+				if (name) await docker(runtime, [
+					"network",
+					"rm",
+					name
+				]).catch(() => void 0);
+			}
+			await Promise.all(running.map(async (entry) => {
+				if (!entry.process) return;
+				if (entry.fixture.stop) await collect(shell(entry.fixture.stop, entry.output));
+				await entry.process.killLocal("SIGTERM");
+				const timer = setTimeout(() => void entry.process?.killLocal("SIGKILL"), STOP_GRACE_MS);
+				timer.unref?.();
+				await entry.process.wait();
+				clearTimeout(timer);
+			}));
+		}
+	};
+}
+//#endregion
+//#region ../runner/src/shell/index.ts
+/**
+* Opens a run's shell where its isolation puts it, starting the project's
+* fixtures first when there are any.
+*/
+async function openRunShell(options) {
+	const { isolation } = options;
+	const home = path.join(path.dirname(options.root), "home");
+	await mkdir(home, { recursive: true });
+	await options.fixtures?.ensure();
+	const fixtures = options.fixtures?.addresses(isolation) ?? {};
+	if (isolation.kind === "container") {
+		const fixturesNetwork = isolation.network === "fixtures" && options.fixtures ? await options.fixtures.network(isolation.runtime) : void 0;
+		const shell = await containerShell({
+			isolation,
+			runId: options.runId,
+			root: options.root,
+			home,
+			projectRoot: options.projectRoot,
+			signal: options.signal,
+			...fixturesNetwork ? { fixturesNetwork } : {}
+		});
+		return {
+			shell,
+			fixtures,
+			close: ({ keep }) => shell.close({ keep })
+		};
+	}
+	const shell = isolation.kind === "sandbox-exec" ? await sandboxShell({
+		isolation,
+		root: options.root,
+		home,
+		projectRoot: options.projectRoot,
+		signal: options.signal
+	}) : hostShell({
+		root: options.root,
+		home,
+		signal: options.signal
+	});
+	return {
+		shell,
+		fixtures,
+		close: () => shell.close()
+	};
+}
+//#endregion
+//#region ../core/src/glob.ts
+/**
+* A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
+* spans segments, and `{a,b}` is an alternation.
+*/
+function globMatcher(pattern) {
+	let source = "";
+	for (let index = 0; index < pattern.length; index++) {
+		const char = pattern[index];
+		if (char === "*" && pattern[index + 1] === "*") {
+			index++;
+			if (pattern[index + 1] === "/") {
+				index++;
+				source += "(?:.*/)?";
+			} else source += ".*";
+		} else if (char === "*") source += "[^/]*";
+		else if (char === "?") source += "[^/]";
+		else if (char === "{") {
+			const end = pattern.indexOf("}", index);
+			if (end === -1) source += "\\{";
+			else {
+				source += `(?:${pattern.slice(index + 1, end).split(",").map((part) => part.replace(/[.+^$()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*")).join("|")})`;
+				index = end;
+			}
+		} else source += char.replace(/[.+^$()|[\]\\]/g, "\\$&");
+	}
+	const expression = new RegExp(`^${source}$`);
+	return (path) => expression.test(path);
+}
+//#endregion
+//#region ../runner/src/snapshot.ts
+const GLOB_CHARS = /[*?{]/;
+function ignoreRule(pattern) {
+	const negated = pattern.startsWith("!");
+	const trimmed = (negated ? pattern.slice(1) : pattern).replace(/^\.\//, "").replace(/\/$/, "");
+	const match = globMatcher(trimmed);
+	if (!trimmed.includes("/")) return {
+		negated,
+		match: (relativePath) => match(path.posix.basename(relativePath)),
+		mayMatchBelow: () => true
+	};
+	const segments = trimmed.split("/");
+	const firstGlob = segments.findIndex((segment) => GLOB_CHARS.test(segment));
+	const prefix = segments.slice(0, firstGlob === -1 ? segments.length : firstGlob).join("/");
+	return {
+		negated,
+		match,
+		mayMatchBelow: (directory) => prefix === "" || prefix.startsWith(`${directory}/`) || prefix === directory || directory.startsWith(`${prefix}/`)
+	};
+}
+/**
+* Ignore rules for candidate snapshots, read like `.gitignore`: globs relative
+* to the workspace root, where a pattern without `/` matches that name at any
+* depth, and later patterns win. `!pattern` re-includes a path; unlike Git, it
+* also works below an ignored directory (`_build` then `!_build/doc/**`).
+*/
+function ignoreRules(patterns) {
+	const rules = patterns.map(ignoreRule);
+	return {
+		/** Whether a path is ignored, given whether its directory was. */
+		ignored(relativePath, parentIgnored = false) {
+			let ignored = parentIgnored;
+			for (const rule of rules) if (rule.match(relativePath)) ignored = !rule.negated;
+			return ignored;
+		},
+		/** Whether an ignored directory must still be walked for re-included paths. */
+		mayReinclude(directory) {
+			return rules.some((rule) => rule.negated && rule.mayMatchBelow(directory));
+		}
+	};
+}
+/**
+* Copies the candidate workspace's regular files into the run report,
+* leaving out ignored paths. Symbolic links and special files are skipped,
+* never followed, so a snapshot cannot read outside the run. An ignored
+* directory is walked only when a `!` pattern could re-include something in
+* it, and is listed only if it ends up containing a snapshotted file.
+*/
+async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAULT_ARTIFACT_EXCLUDE) {
+	const artifacts = [];
+	const rules = ignoreRules(ignore);
+	/** Returns whether anything below `directory` was snapshotted. */
+	async function visit(directory, relativeDirectory, directoryIgnored) {
+		const entries = await readdir(directory, { withFileTypes: true });
+		entries.sort((left, right) => left.name.localeCompare(right.name));
+		let kept = false;
+		for (const entry of entries) {
+			const absolutePath = path.join(directory, entry.name);
+			const relativePath = path.join(relativeDirectory, entry.name);
+			const posixPath = relativePath.split(path.sep).join("/");
+			const ignored = rules.ignored(posixPath, directoryIgnored);
+			const stat = await lstat(absolutePath);
+			if (stat.isSymbolicLink()) continue;
+			if (stat.isDirectory()) {
+				if (ignored && !rules.mayReinclude(posixPath)) continue;
+				const entryIndex = artifacts.length;
+				const keptBelow = await visit(absolutePath, relativePath, ignored);
+				if (!ignored || keptBelow) {
+					artifacts.splice(entryIndex, 0, {
+						path: path.join("candidate", relativePath),
+						kind: "directory"
+					});
+					kept = true;
+				}
+				continue;
+			}
+			if (ignored || !stat.isFile()) continue;
+			const data = await readFile(absolutePath);
+			const artifactPath = path.join("candidate", relativePath);
+			await writer.writeArtifact(artifactPath, data);
+			artifacts.push({
+				path: artifactPath,
+				kind: "file",
+				size: data.byteLength
+			});
+			kept = true;
+		}
+		return kept;
+	}
+	await visit(candidateRoot, "", false);
+	return artifacts;
+}
+//#endregion
+//#region ../runner/src/datasets.ts
+const loads = /* @__PURE__ */ new WeakMap();
+/**
+* Loads a dataset at most once per process and project root. A failed load is
+* forgotten, so the next run that needs the dataset tries again.
+*/
+function loadDataset(dataset, root, signal) {
+	let byRoot = loads.get(dataset);
+	if (!byRoot) loads.set(dataset, byRoot = /* @__PURE__ */ new Map());
+	const cached = byRoot.get(root);
+	if (cached) return cached;
+	const loading = loadUncached(dataset, root, signal);
+	byRoot.set(root, loading);
+	loading.catch(() => {
+		if (byRoot.get(root) === loading) byRoot.delete(root);
+	});
+	return loading;
+}
+async function loadUncached(dataset, root, signal) {
+	const { source } = dataset;
+	if (source.kind === "computed") {
+		const loaded = await source.onLoad({
+			root,
+			signal
+		});
+		if (typeof loaded !== "object" || loaded === null) throw new Error(`Dataset ${dataset.id} onLoad must return { files?, value? }`);
+		return loaded;
+	}
+	return {
+		...source.files !== void 0 ? { files: source.files } : {},
+		...source.valueFile !== void 0 ? { value: await readValueFile(dataset.id, path.resolve(root, source.valueFile)) } : source.value !== void 0 ? { value: source.value } : {}
+	};
+}
+/** `.json` is one value; `.jsonl` is an array of one JSON object per non-blank line. */
+async function readValueFile(id, file) {
+	const text = await readFile(file, "utf8").catch((cause) => {
+		throw new Error(`Dataset ${id} cannot read ${file}: ${String(cause)}`);
+	});
+	if (file.endsWith(".jsonl")) return text.split("\n").flatMap((line, index) => {
+		if (!line.trim()) return [];
+		try {
+			const row = JSON.parse(line);
+			if (typeof row !== "object" || row === null || Array.isArray(row)) throw new Error("expected a JSON object");
+			return [row];
+		} catch (cause) {
+			throw new Error(`Dataset ${id} ${file}:${index + 1}: ${cause instanceof Error ? cause.message : String(cause)}`);
+		}
+	});
+	if (file.endsWith(".json")) try {
+		return JSON.parse(text);
+	} catch (cause) {
+		throw new Error(`Dataset ${id} ${file}: ${cause instanceof Error ? cause.message : String(cause)}`);
+	}
+	throw new Error(`Dataset ${id} valueFile must be a .json or .jsonl file: ${file}`);
+}
+/** Loads (cached), derives this run's content with `onRun`, and resolves it. */
+async function prepareDatasets(datasets, root, run) {
+	const prepared = [];
+	for (const dataset of datasets) {
+		const loaded = await loadDataset(dataset, root, run.signal);
+		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, run) : loaded;
+		if (typeof content !== "object" || content === null) throw new Error(`Dataset ${dataset.id} onRun must return { files?, value? }`);
+		prepared.push(await resolveContent(dataset, content, root));
+	}
+	return prepared;
+}
+async function resolveContent(dataset, content, root) {
+	const label = `Dataset ${dataset.id}`;
+	const value = content.value === void 0 ? void 0 : toJson(content.value, `datasets.${dataset.id}.value`);
+	let files = { kind: "none" };
+	if (typeof content.files === "string") {
+		const source = path.resolve(root, content.files);
+		if (!(await stat(source).catch(() => void 0))?.isDirectory()) throw new Error(`${label} files directory does not exist: ${content.files}`);
+		files = {
+			kind: "directory",
+			source
+		};
+	} else if (content.files !== void 0) files = {
+		kind: "entries",
+		entries: Object.entries(content.files).map(([file, source]) => preparedFile(relativeDatasetPath(file, `${label} file`), source, root, label))
+	};
+	return {
+		dataset,
+		files,
+		...value === void 0 ? {} : { value },
+		digest: await digest(files, value)
+	};
+}
+function preparedFile(file, source, root, label) {
+	if (typeof source === "string") return {
+		path: file,
+		kind: "bytes",
+		data: new TextEncoder().encode(source)
+	};
+	if (source instanceof Uint8Array) return {
+		path: file,
+		kind: "bytes",
+		data: source
+	};
+	if (typeof source === "object" && source !== null && typeof source.file === "string") return {
+		path: file,
+		kind: "file",
+		source: path.resolve(root, source.file)
+	};
+	throw new Error(`${label} file ${file} must be a string, bytes, or { file: 'path' }`);
+}
+/** Lists a directory's files as sorted relative POSIX paths. */
+async function listFiles(directory, prefix = "") {
+	const entries = await readdir(path.join(directory, prefix), { withFileTypes: true });
+	const files = [];
+	for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+		const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+		if (entry.isDirectory()) files.push(...await listFiles(directory, relative));
+		else if (entry.isFile()) files.push(relative);
+	}
+	return files;
+}
+async function digest(files, value) {
+	const hash = createHash("sha256");
+	const addFile = (file, data) => {
+		hash.update(`file ${file} ${data.byteLength}\n`);
+		hash.update(data);
+	};
+	if (files.kind === "directory") for (const file of await listFiles(files.source)) addFile(file, await readFile(path.join(files.source, file)));
+	else if (files.kind === "entries") for (const entry of [...files.entries].sort((a, b) => a.path.localeCompare(b.path))) addFile(entry.path, entry.kind === "bytes" ? entry.data : await readFile(entry.source));
+	if (value !== void 0) hash.update(`value ${canonicalParameters(value)}\n`);
+	return `sha256:${hash.digest("hex")}`;
+}
+function runDatasets(prepared) {
+	return prepared.map(({ dataset, digest }) => ({
+		id: dataset.id,
+		visibility: dataset.visibility,
+		source: dataset.source.kind,
+		digest
+	}));
+}
+/**
+* The files a static dataset places in every workspace, relative to its
+* mount. Computed datasets return `undefined`: their files only exist once
+* `onLoad` runs, and browsing must never run project code.
+*/
+async function staticDatasetFiles(dataset, root) {
+	const { source } = dataset;
+	if (source.kind === "computed") return void 0;
+	if (source.files === void 0) return [];
+	if (typeof source.files === "string") {
+		const directory = path.resolve(root, source.files);
+		return Promise.all((await listFiles(directory)).map(async (file) => ({
+			path: file,
+			size: (await stat(path.join(directory, file))).size
+		})));
+	}
+	return Promise.all(Object.entries(source.files).sort(([a], [b]) => a.localeCompare(b)).map(async ([file, content]) => ({
+		path: file,
+		size: typeof content === "string" ? new TextEncoder().encode(content).byteLength : content instanceof Uint8Array ? content.byteLength : (await stat(path.resolve(root, content.file))).size
+	})));
+}
+/** One file of a static dataset, by its path below the mount. */
+async function staticDatasetFile(dataset, root, file) {
+	const { source } = dataset;
+	const missing = () => /* @__PURE__ */ new Error(`Dataset ${dataset.id} has no file ${JSON.stringify(file)}`);
+	if (source.kind === "computed" || source.files === void 0) throw missing();
+	if (typeof source.files === "string") {
+		const directory = path.resolve(root, source.files);
+		const resolved = path.resolve(directory, file);
+		if (!resolved.startsWith(`${directory}${path.sep}`)) throw missing();
+		if (!(await stat(resolved).catch(() => void 0))?.isFile()) throw missing();
+		return readFile(resolved);
+	}
+	if (!Object.hasOwn(source.files, file)) throw missing();
+	const content = source.files[file];
+	if (typeof content === "string") return new TextEncoder().encode(content);
+	if (content instanceof Uint8Array) return new Uint8Array(content);
+	return readFile(path.resolve(root, content.file));
+}
+//#endregion
+//#region ../runner/src/workspace.ts
+/** Where a run's persistent workspace lives below `root`. */
+function runWorkspacePath(root, runId) {
+	return path.join(path.resolve(root), runId.replace(/[^a-zA-Z0-9._-]/g, "_"));
+}
+/**
+* Creates isolated candidate and evaluator directories and writes each
+* dataset's files below its mount, on the side its visibility allows.
+* Two datasets writing the same file is an error, never an overwrite.
+*/
+async function createRunWorkspace(runId, datasets = [], persistentRoot) {
+	const root = persistentRoot ? runWorkspacePath(persistentRoot, runId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-run-"));
+	const persistent = persistentRoot !== void 0;
+	const artifacts = {
+		candidate: { root: path.join(root, "candidate") },
+		evaluator: { root: path.join(root, "evaluator") }
+	};
+	try {
+		await Promise.all([mkdir(artifacts.candidate.root, { recursive: true }), mkdir(artifacts.evaluator.root, { recursive: true })]);
+		for (const prepared of datasets) {
+			const side = prepared.dataset.visibility === "candidate" ? artifacts.candidate.root : artifacts.evaluator.root;
+			await writeDatasetFiles(prepared, path.join(side, prepared.dataset.mount));
+		}
+		return {
+			artifacts,
+			cleanup: async (keep = false) => {
+				if (persistent && keep) return;
+				await rm(root, {
+					recursive: true,
+					force: true
+				});
+			}
+		};
+	} catch (error) {
+		await rm(root, {
+			recursive: true,
+			force: true
+		});
+		throw error;
+	}
+}
+async function writeDatasetFiles(prepared, target) {
+	const { files, dataset } = prepared;
+	if (files.kind === "none") return;
+	await mkdir(target, { recursive: true });
+	const entries = files.kind === "directory" ? (await listFiles(files.source)).map((file) => ({
+		path: file,
+		kind: "file",
+		source: path.join(files.source, file)
+	})) : files.entries;
+	for (const entry of entries) await writeEntry(dataset.id, target, entry);
+}
+async function writeEntry(datasetId, target, entry) {
+	const destination = path.join(target, ...entry.path.split("/"));
+	await mkdir(path.dirname(destination), { recursive: true });
+	await (entry.kind === "bytes" ? writeFile(destination, entry.data, { flag: "wx" }) : copyFile(entry.source, destination, 1)).catch((cause) => {
+		throw collision(datasetId, entry.path, cause);
+	});
+}
+function collision(datasetId, file, cause) {
+	return cause.code === "EEXIST" ? /* @__PURE__ */ new Error(`Dataset ${datasetId} would overwrite ${file}, which another dataset already wrote`) : cause instanceof Error ? cause : new Error(String(cause));
+}
+//#endregion
+//#region ../runner/src/checkpoint-lookup.ts
+/**
+* `checkpoint(rule)` for a scoring context: the latest result of the steps
+* that are this very rule. A rule outside the transcript is a mistake in the
+* scenario, so it throws rather than reading as "not run".
+*/
+function checkpointLookup(transcript, results) {
+	return ((rule) => {
+		const steps = new Set(transcript.flatMap((step, index) => step === rule ? [index] : []));
+		if (!steps.size) throw new Error(`checkpoint(${JSON.stringify(rule.name)}) is not a checkpoint in this scenario's transcript; pass the same rule object the transcript uses`);
+		for (let index = results.length - 1; index >= 0; index--) if (steps.has(results[index].step)) return results[index];
+	});
+}
+//#endregion
+//#region ../runner/src/score.ts
+const inUnitInterval = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+/**
+* Criteria must be well formed before they can stand for the score. With
+* declared criteria, each takes its declared weight unless it sets one, a
+* declared one left out is not reached, and an undeclared one is an error,
+* so the checks shown before a run are the checks recorded after it.
+*/
+function checkedCriteria(criteria, declared) {
+	if (!Array.isArray(criteria) || !criteria.length && !declared?.length) throw new Error("criteria must be a non-empty array");
+	const seen = /* @__PURE__ */ new Set();
+	const results = criteria.map((entry, index) => {
+		if (typeof entry?.name !== "string" || !entry.name.trim()) throw new Error(`criteria[${index}] needs a name`);
+		const { name } = entry;
+		if (seen.has(name)) throw new Error(`Criterion ${name} is listed twice`);
+		seen.add(name);
+		const declaration = declared?.find((criterion) => criterion.name === name);
+		if (declared && !declaration) throw new Error(`Criterion ${name} is not declared; declared: ${declared.map((criterion) => criterion.name).join(", ")}`);
+		const weight = entry.weight ?? declaration?.weight;
+		if (weight !== void 0 && !(Number.isFinite(weight) && weight > 0)) throw new Error(`Criterion ${name} weight must be a positive number; received ${String(weight)}`);
+		const explanation = entry.explanation;
+		if (explanation !== void 0 && typeof explanation !== "string") throw new Error(`Criterion ${name} explanation must be a string`);
+		const common = {
+			name,
+			...weight !== void 0 ? { weight } : {},
+			...explanation?.trim() ? { explanation: explanation.trim() } : {}
+		};
+		if ("reached" in entry) {
+			if (entry.reached !== false) throw new Error(`Criterion ${name}: reached can only be false; leave it out for a check that ran`);
+			return {
+				...common,
+				reached: false
+			};
+		}
+		const value = entry.value;
+		if (!inUnitInterval(value)) throw new Error(`Criterion ${name} value must be a number from 0 to 1; received ${String(value)}`);
+		const evidence = entry.evidence;
+		return {
+			...common,
+			value,
+			...evidence !== void 0 ? { evidence: toJson(evidence, `criteria.${name}.evidence`) } : {}
+		};
+	});
+	const missing = (declared ?? []).filter((criterion) => !seen.has(criterion.name)).map((criterion) => ({
+		name: criterion.name,
+		weight: criterion.weight,
+		reached: false
+	}));
+	return [...results, ...missing];
+}
+/**
+* Shared normalization for inline and final predicate and judge rules.
+* Without a `value`, the weighted mean of `criteria` is the value. `passAt`
+* turns the value into a verdict, and is recorded with it; an explicit
+* `passed` wins.
+*/
+function normalizeScore(value, passAt, declared) {
+	if (typeof value === "object" && value !== null && "skipped" in value && value.skipped === true) return {
+		skipped: true,
+		explanation: value.reason
+	};
+	const normalized = typeof value === "boolean" ? { value: Number(value) } : typeof value === "number" ? { value } : value;
+	if (!normalized || typeof normalized !== "object") throw new Error(`A scorer must return a boolean, a number from 0 to 1, or a score object; received ${String(value)}`);
+	const criteria = normalized.criteria === void 0 ? void 0 : checkedCriteria(normalized.criteria, declared);
+	const score = normalized.value ?? (criteria ? weightedMean(criteria) : void 0);
+	if (!inUnitInterval(score)) throw new Error(`Score value must be a finite number from 0 to 1; received ${String(score)}`);
+	if (normalized.judge?.usage) {
+		const { usage } = normalized.judge;
+		if (Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
+	}
+	const mark = passAt ?? 1;
+	return {
+		value: score,
+		passed: normalized.passed ?? score >= mark,
+		passAt: mark,
+		...normalized.explanation ? { explanation: normalized.explanation } : {},
+		...normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
+		...normalized.choice ? { choice: normalized.choice } : {},
+		...criteria ? { criteria } : {},
+		...normalized.judge !== void 0 ? { judge: normalized.judge } : {}
+	};
+}
+//#endregion
+//#region ../runner/src/judge-agent.ts
+const failure$2 = (name, cause) => new RuleExecutionError({
+	name,
+	cause,
+	message: cause instanceof Error ? cause.message : String(cause)
+});
+/** A long tool result or reply is cut to this many characters in a judge's conversation. */
+const MAX_PART = 4e3;
+/**
+* Drives one judge agent session for one assessment: builds the prompt from
+* the rule, sends it, reads the answer in the rule's format, and asks once
+* more if it cannot. Its session is its own transcript, never the agent's.
+*/
+function runJudgeAgent(rule, agent, input, placement) {
+	return Effect.tryPromise({
+		try: () => judgeOnce(rule, agent, input, placement),
+		catch: (cause) => cause instanceof RuleExecutionError ? cause : failure$2(rule.name, cause)
+	});
+}
+let judgeSessions = 0;
+async function judgeOnce(rule, agent, input, placement) {
+	const inputs = judgeInputNames(rule);
+	const values = await inputValues(rule, input, placement);
+	const prompt = judgePrompt({
+		rubric: rule.rubric,
+		answer: rule.answer,
+		inputs,
+		values
+	});
+	const now = () => /* @__PURE__ */ new Date();
+	const transcript = new RunTranscript({
+		id: `${input.context.runId}-judge-${++judgeSessions}`,
+		agent: {
+			id: agent.identity?.id ?? "judge",
+			...agent.identity?.version ? { version: agent.identity.version } : {}
+		},
+		runId: input.context.runId,
+		scenarioId: input.context.evalId,
+		now
+	});
+	const context = {
+		...input.context,
+		data: input.data,
+		workspace: input.artifacts.evaluator,
+		evaluatorWorkspace: input.artifacts.evaluator
+	};
+	const session = await agent.start({
+		context,
+		attach: input.attach,
+		transcript: transcript.writer()
+	});
+	let reading;
+	let reply;
+	let attempts = 0;
+	try {
+		let message = prompt;
+		let problem = "";
+		while (attempts < 2 && !reading) {
+			attempts += 1;
+			const from = transcript.length;
+			transcript.scriptUser(message);
+			const result = await session.send(message);
+			if (result !== void 0) transcript.writeTurnResult(result, attempts);
+			transcript.endTurn();
+			reply = replyIn(transcript.since(from));
+			if (reply === void 0) problem = "it did not reply";
+			else try {
+				reading = readJudgeAnswer(rule.answer, reply);
+			} catch (error) {
+				if (!(error instanceof UnreadableAnswer)) throw error;
+				problem = error.message;
+			}
+			message = `Your answer could not be read: ${problem}. Answer again, only in this format.\n\n${answerInstruction(rule.answer)}`;
+		}
+		if (!reading) throw new Error(`Judge ${rule.name}'s answer could not be read after ${attempts} tries: ${problem}${reply === void 0 ? "" : `. Its last reply: ${preview(reply)}`}`);
+	} finally {
+		await session.close?.();
+	}
+	transcript.seal();
+	const problems = transcript.problems();
+	if (problems.length) throw new Error(`Judge ${rule.name}'s transcript is not valid ATXF: ${problems.slice(0, 3).join("; ")}`);
+	const recorded = transcript.transcript;
+	const model = lastOf(recorded.records, (record) => record.type === "message" && record.role === "assistant" && record.model !== void 0);
+	const usage = transcriptUsage(recorded);
+	const judge = {
+		...agent.identity ? { agent: agent.identity } : {},
+		...model?.type === "message" && model.role === "assistant" && model.model ? { model: modelName(model.model) } : {},
+		...usage ? { usage } : {},
+		prompt,
+		...reply !== void 0 ? { reply } : {},
+		...attempts > 1 ? { attempts } : {},
+		transcript: recorded
+	};
+	return { value: {
+		...reading,
+		judge
+	} };
+}
+/** Each input's text: what the rule's function returned, or the defaults. */
+async function inputValues(rule, input, placement) {
+	if (!rule.inputs) return {
+		conversation: conversationText(input.transcript, placement),
+		reply: text(input.output)
+	};
+	const values = {};
+	for (const [name, read] of Object.entries(rule.inputs)) values[name] = text(await read(input));
+	return values;
+}
+/**
+* The agent's conversation as plain lines, for a judge to read: this turn
+* (from the last user message) at a checkpoint, else all of it.
+*/
+function conversationText(transcript, placement) {
+	let path = activePath(transcript);
+	if (placement === "transcript") {
+		let start = 0;
+		path.forEach((record, index) => {
+			if (record.type === "message" && record.role === "user") start = index;
+		});
+		path = path.slice(start);
+	}
+	const lines = [];
+	for (const record of path) {
+		if (record.type !== "message") continue;
+		if (record.role === "tool") lines.push(`Tool result ${record.toolName ?? record.toolCallId}: ${cut(contentText(record.content))}`);
+		else if (record.role === "assistant") {
+			const said = contentText(record.content);
+			if (said) lines.push(`Assistant: ${cut(said)}`);
+			for (const block of record.content) if (block.type === "tool_call") lines.push(`Tool call ${block.name}: ${cut(block.input ?? JSON.stringify(block.arguments ?? {}, null, 2))}`);
+		} else lines.push(`${capitalize(record.role)}: ${cut(contentText(record.content))}`);
+	}
+	return lines.join("\n") || "(nothing happened)";
+}
+/** The judge's answer in one attempt: the text of its last reply. */
+function replyIn(records) {
+	const assistant = lastOf(records, (record) => record.type === "message" && record.role === "assistant");
+	if (!assistant) return void 0;
+	return contentText(assistant.content) || void 0;
+}
+function text(value) {
+	if (value === void 0) return "(none)";
+	return typeof value === "string" ? value : JSON.stringify(value, null, 2);
+}
+function cut(value) {
+	return value.length > MAX_PART ? `${value.slice(0, MAX_PART)}… (${value.length - MAX_PART} more characters)` : value;
+}
+function preview(value) {
+	const shown = text(value).replace(/\s+/g, " ");
+	return shown.length > 300 ? `${shown.slice(0, 300)}…` : shown;
+}
+/** The last element matching, scanning from the end (no ES2023 lib here). */
+function lastOf(items, match) {
+	for (let index = items.length - 1; index >= 0; index -= 1) if (match(items[index])) return items[index];
+}
+const capitalize = (value) => value.charAt(0).toUpperCase() + value.slice(1);
+//#endregion
+//#region ../runner/src/evaluate-rule.ts
+const failure$1 = (name, cause) => new RuleExecutionError({
+	name,
+	cause,
+	message: cause instanceof Error ? cause.message : String(cause)
+});
+/** Both placements call the same rule executor; only input timing and persistence differ. */
+function evaluateRule(rule, input, defaultJudge, placement) {
+	return Effect.gen(function* () {
+		const judgeAgent = rule.kind === "judge" ? rule.agent ?? defaultJudge : void 0;
+		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("This judge has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts")));
+		const evaluated = rule.kind === "predicate" ? yield* Effect.tryPromise({
+			try: async () => rule.run(input),
+			catch: (cause) => failure$1(rule.name, cause)
+		}) : (yield* runJudgeAgent(rule, judgeAgent, input, placement)).value;
+		const score = yield* Effect.try({
+			try: () => normalizeScore(evaluated, rule.passAt, rule.kind === "predicate" ? rule.criteria : void 0),
+			catch: (cause) => failure$1(rule.name, cause)
+		});
+		const validated = yield* Schema.decodeUnknown(ScoreResultSchema)({
+			name: rule.name,
+			kind: rule.kind,
+			durationMs: 0,
+			...score
+		}).pipe(Effect.mapError((cause) => failure$1(rule.name, cause)));
+		if (validated.skipped) return {
+			skipped: true,
+			...validated.explanation === void 0 ? {} : { explanation: validated.explanation }
+		};
+		const { name: _name, kind: _kind, durationMs: _duration, ...result } = validated;
+		return result;
+	});
+}
+//#endregion
+//#region ../runner/src/turn.ts
+/** Freeze one turn's records at send() completion, before checkpoint events arrive. */
+function turnView(records, userStepIndex) {
+	const tree = records.filter(isTreeRecord);
+	const assistantMessages = tree.filter((record) => record.type === "message" && record.role === "assistant");
+	const calls = callsIn(tree);
+	const resultCount = /* @__PURE__ */ new Map();
+	for (const record of tree) if (record.type === "message" && record.role === "tool") resultCount.set(record.toolCallId, (resultCount.get(record.toolCallId) ?? 0) + 1);
+	const last = assistantMessages.at(-1);
+	const lastText = last ? contentText(last.content) : void 0;
+	return {
+		userStepIndex,
+		records: tree,
+		assistantMessages,
+		toolCalls: calls.map(({ message, call, result }) => {
+			const ambiguous = calls.filter((entry) => entry.call.id === call.id).length !== 1 || (resultCount.get(call.id) ?? 0) > 1;
+			return {
+				messageId: message.id,
+				id: call.id,
+				name: call.name,
+				arguments: toJson(call.arguments ?? {}, "tool arguments"),
+				resultObservation: ambiguous ? "ambiguous" : result ? "observed" : "absent",
+				...!ambiguous && result ? { result: toJson(toolResultValue(result), "tool result") } : {}
+			};
+		}),
+		...lastText ? { lastAssistantText: lastText } : {}
+	};
+}
+/** Exact JSON matching: object keys do not matter, array order does. */
+function matchingToolCall(turn, name, args) {
+	const expected = args === void 0 ? void 0 : canonicalParameters(args);
+	return turn.toolCalls.find((call) => call.name === name && (expected === void 0 || canonicalParameters(call.arguments) === expected));
+}
+//#endregion
+//#region ../runner/src/checkpoint.ts
+const failure = (step, name, cause) => new CheckpointExecutionError({
+	step,
+	name,
+	cause,
+	message: cause instanceof Error ? cause.message : String(cause)
+});
+/** An assertion failure is a successful Effect containing a failed result; only execution errors enter the error channel. */
+function evaluateCheckpoint(step, index, input, now, judgeAgent) {
+	const annotations = {
+		evalId: input.context.evalId,
+		runId: input.context.runId,
+		step: index,
+		checkpoint: step.name
+	};
+	return Effect.gen(function* () {
+		const started = now();
+		yield* Effect.logDebug("checkpoint started");
+		const match = step.kind === "expect-tool-call" ? yield* Effect.try({
+			try: () => matchingToolCall(input.turn, step.expected.name, step.expected.arguments),
+			catch: (cause) => failure(index, step.name, cause)
+		}) : void 0;
+		const score = step.kind === "expect-tool-call" ? yield* Effect.try({
+			try: () => normalizeScore(Boolean(match)),
+			catch: (cause) => failure(index, step.name, cause)
+		}) : yield* evaluateRule(step, input, judgeAgent, "transcript").pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
+		const status = score.skipped ? "skipped" : score.passed ? "passed" : "failed";
+		const { skipped: _, ...recorded } = score;
+		const result = yield* Schema.decodeUnknown(CheckpointResultSchema)({
+			step: index,
+			kind: step.kind,
+			name: step.name,
+			status,
+			durationMs: now().getTime() - started.getTime(),
+			...recorded,
+			...match ? { matchedToolCall: {
+				messageId: match.messageId,
+				id: match.id
+			} } : {}
+		}).pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
+		yield* Effect.logDebug("checkpoint completed").pipe(Effect.annotateLogs({ status: result.status }));
+		return {
+			...result,
+			status
+		};
+	}).pipe(Effect.tapError((error) => Effect.logDebug("checkpoint execution error").pipe(Effect.annotateLogs({ errorName: error._tag }))), Effect.annotateLogs(annotations));
+}
+//#endregion
+//#region ../runner/src/effect-boundary.ts
+/** Keep tagged Effect failures intact when bridging into the legacy async reporting lifecycle. Defects still reject. */
+async function runBoundary(effect) {
+	const outcome = await Effect.runPromise(Effect.either(effect));
+	if (Either.isLeft(outcome)) throw outcome.left;
+	return outcome.right;
+}
+//#endregion
+//#region ../runner/src/transcript.ts
+async function executeTranscript(definition, session, input) {
+	const { now, emitRunner, progress } = input;
+	let turn = 0;
+	for (const [index, step] of definition.transcript.entries()) {
+		if (input.signal.aborted) throw abortError(input.signal);
+		const message = step.kind === "user" ? userMessage(step, index, input) : void 0;
+		await emitRunner({
+			kind: "transcript-step-started",
+			step: index,
+			...message === void 0 ? {} : { message },
+			timestamp: now().toISOString()
+		});
+		if (message !== void 0) {
+			const cursor = input.transcript.length;
+			turn++;
+			input.transcript.scriptUser(message);
+			const result = await sendUserTurn(session, message, input.signal);
+			if (result !== void 0) input.transcript.writeTurnResult(result, turn);
+			input.transcript.endTurn();
+			progress.lastTurn = turnView(input.transcript.since(cursor), index);
+		} else if (step.kind === "predicate" || step.kind === "judge" || step.kind === "expect-tool-call") {
+			const result = await runCheckpoint(definition, step, index, input);
+			progress.checkpoints.push(result);
+			await emitRunner({
+				kind: "checkpoint-completed",
+				step: index,
+				status: result.status,
+				timestamp: now().toISOString()
+			});
+			await emitRunner({
+				kind: "transcript-step-completed",
+				step: index,
+				timestamp: now().toISOString()
+			});
+			if (result.status === "failed" && definition.policy?.failfast) {
+				progress.stoppedEarly = true;
+				await skipRemainingSteps(definition, index, input);
+				break;
+			}
+			continue;
+		} else throw new Error(`Transcript step kind "${step.kind}" is not executable yet`);
+		await emitRunner({
+			kind: "transcript-step-completed",
+			step: index,
+			timestamp: now().toISOString()
+		});
+	}
+}
+/** A computed message sees the parameters and candidate data the agent also sees. */
+function userMessage(step, index, { context, randomSeed }) {
+	if (typeof step.message === "string") return step.message;
+	const message = step.message({
+		parameters: context.parameters ?? {},
+		data: context.data,
+		seed: randomSeed,
+		...context.batch ? { batch: context.batch } : {}
+	});
+	if (typeof message !== "string" || !message.trim()) throw new Error(`The user message at step ${index + 1} must compute a non-empty string`);
+	return message;
+}
+async function sendUserTurn(session, message, signal) {
+	return runBoundary(Effect.tryPromise({
+		try: () => unlessAborted(() => session.send(message), signal, LIMIT_GRACE_MS),
+		catch: (cause) => cause instanceof ReportError || cause instanceof CancelledError || cause instanceof LimitReachedError ? cause : new AutExecutionError({
+			cause,
+			message: cause instanceof Error ? cause.message : String(cause)
+		})
+	}));
+}
+async function runCheckpoint(definition, step, index, input) {
+	const { now, emitRunner, progress } = input;
+	if (!progress.lastTurn) throw new Error("Checkpoint requires a preceding user step");
+	const started = now();
+	await emitRunner({
+		kind: "checkpoint-started",
+		step: index,
+		name: step.name,
+		timestamp: started.toISOString()
+	});
+	const tracked = trackAttachments(input.attach);
+	return runBoundary(evaluateCheckpoint(step, index, {
+		context: input.context,
+		artifacts: input.artifacts,
+		transcript: input.transcript.transcript,
+		subagents: input.transcript.subagents,
+		runEvents: [...input.runEvents],
+		turn: progress.lastTurn,
+		...turnOutput(progress.lastTurn),
+		data: input.data,
+		checkpoints: [...progress.checkpoints],
+		checkpoint: checkpointLookup(definition.transcript, [...progress.checkpoints]),
+		attach: tracked.attach
+	}, now, input.defaultJudge)).then((result) => ({
+		...result,
+		...attachedFields(tracked.attached)
+	})).catch(async (error) => {
+		const recorded = recordError(error);
+		progress.checkpoints.push({
+			step: index,
+			kind: step.kind,
+			name: step.name,
+			status: "error",
+			durationMs: now().getTime() - started.getTime(),
+			error: recorded,
+			...attachedFields(tracked.attached)
+		});
+		try {
+			await emitRunner({
+				kind: "checkpoint-error",
+				step: index,
+				error: recorded,
+				timestamp: now().toISOString()
+			});
+		} catch {}
+		throw error;
+	});
+}
+async function skipRemainingSteps(definition, after, { now, emitRunner, progress }) {
+	for (let index = after + 1; index < definition.transcript.length; index++) {
+		await emitRunner({
+			kind: "transcript-step-skipped",
+			step: index,
+			timestamp: now().toISOString()
+		});
+		const step = definition.transcript[index];
+		if (step.kind === "predicate" || step.kind === "judge" || step.kind === "expect-tool-call") progress.checkpoints.push({
+			step: index,
+			kind: step.kind,
+			name: step.name,
+			status: "skipped"
+		});
+	}
+}
+/** The scorer-facing `output`: the text of the turn's last assistant message. */
+function turnOutput(turn) {
+	return turn?.lastAssistantText === void 0 ? {} : { output: turn.lastAssistantText };
+}
+//#endregion
+//#region ../runner/src/final-scoring.ts
+function scoreSummary(results, checkpoints, skippedScorers, metrics = []) {
+	const valid = results.filter((result) => result.value !== void 0 && !result.skipped);
+	const hasError = results.some((result) => result.error !== void 0);
+	return {
+		results,
+		...metrics.length ? { metrics } : {},
+		...checkpoints.length ? { checkpoints } : {},
+		...skippedScorers.length ? { skippedScorers } : {},
+		...valid.length === 0 ? {} : { overall: valid.reduce((sum, result) => sum + (result.value ?? 0), 0) / valid.length },
+		passed: !hasError && results.every((result) => result.skipped || result.passed === true) && checkpoints.every((result) => result.status === "passed" || result.status === "skipped") && !skippedScorers.length
+	};
+}
+/** What final scorers and metrics receive. */
+function scoringContext(input) {
+	return {
+		context: input.context,
+		artifacts: input.workspace.artifacts,
+		transcript: input.agentTranscript.transcript,
+		subagents: input.agentTranscript.subagents,
+		runEvents: [...input.runEvents],
+		...input.lastTurn ? { turn: input.lastTurn } : {},
+		...turnOutput(input.lastTurn),
+		data: input.data,
+		checkpoints: input.checkpoints,
+		checkpoint: checkpointLookup(input.transcript, input.checkpoints),
+		attach: input.attach
+	};
+}
+/** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
+async function evaluateFinalScorers(definition, input) {
+	const { context, workspace, results, skippedScorers, now, emitRunner } = input;
+	for (const scorer of definition.scoring) {
+		if (input.cancelled || input.reportingFailed || !context || !workspace || (input.primaryError || input.stoppedEarly || input.limited) && (!("supportsPartial" in scorer) || !scorer.supportsPartial)) {
+			skippedScorers.push(scorer.name);
+			continue;
+		}
+		const started = now();
+		await emitRunner({
+			kind: "scorer-started",
+			scorer: scorer.name,
+			timestamp: started.toISOString()
+		});
+		let result;
+		const tracked = trackAttachments(input.attach);
+		try {
+			const score = await runBoundary(evaluateRule(scorer, scoringContext({
+				...input,
+				attach: tracked.attach,
+				context,
+				workspace,
+				transcript: definition.transcript
+			}), input.defaultJudge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
+				cause,
+				message: cause.message
+			}))));
+			result = {
+				name: scorer.name,
+				kind: scorer.kind,
+				durationMs: now().getTime() - started.getTime(),
+				...score,
+				...attachedFields(tracked.attached)
+			};
+			results.push(result);
+		} catch (error) {
+			const recorded = recordError(error);
+			results.push({
+				name: scorer.name,
+				kind: scorer.kind,
+				durationMs: now().getTime() - started.getTime(),
+				error: recorded,
+				passed: false,
+				...attachedFields(tracked.attached)
+			});
+			await emitRunner({
+				kind: "scorer-failed",
+				scorer: scorer.name,
+				error: recorded,
+				timestamp: now().toISOString()
+			});
+			continue;
+		}
+		await emitRunner({
+			kind: "scorer-completed",
+			scorer: scorer.name,
+			...result.skipped ? { skipped: true } : { value: result.value ?? 0 },
+			timestamp: now().toISOString()
+		});
+	}
+}
+//#endregion
+//#region ../runner/src/metrics.ts
+/**
+* What EvalKit measures for every run: from its transcript and events, the
+* agent's time (from the first record or event to the last one before
+* final scoring), its token metrics when it reported usage (`tokens`
+* without cache reads, `cached tokens`, `output tokens`, `cache hit
+* rate`), its steps (assistant messages) and tool calls, sub-agents'
+* included; from its scores and checkpoints, the tokens its judges used,
+* kept apart from the agent's.
+*/
+function builtInMetrics(transcript, runEvents, verdicts = [], subagents = []) {
+	const times = [...transcript.records.map((record) => record.time), ...runEvents.filter((event) => !event.kind.startsWith("scorer-")).map((event) => event.timestamp)].map((time) => Date.parse(time)).filter((time) => !Number.isNaN(time));
+	const metrics = [];
+	if (times.length) metrics.push({
+		name: "agent time",
+		value: Math.max(...times) - Math.min(...times),
+		unit: "ms",
+		better: "lower",
+		builtIn: true
+	});
+	const all = [transcript, ...subagents];
+	metrics.push(...usageMetrics(all.flatMap(reportedUsages)));
+	let steps = 0;
+	let calls = 0;
+	for (const record of all.flatMap((entry) => entry.records)) if (record.type === "message" && record.role === "assistant") {
+		steps += 1;
+		calls += record.content.filter((block) => block.type === "tool_call").length;
+	}
+	metrics.push({
+		name: "steps",
+		value: steps,
+		better: "lower",
+		builtIn: true
+	}, {
+		name: "tool calls",
+		value: calls,
+		builtIn: true
+	});
+	const judgeUsages = verdicts.flatMap((verdict) => verdict.judge?.transcript ? reportedUsages(verdict.judge.transcript) : verdict.judge?.usage ? [verdict.judge.usage] : []);
+	const judgeTokens = usageMetrics(judgeUsages).find((metric) => metric.name === "tokens");
+	if (judgeTokens) metrics.push({
+		...judgeTokens,
+		name: "judge tokens"
+	});
+	return metrics;
+}
+/** Runs the scenario's metrics; a failing metric records its error and the rest still run. */
+async function evaluateMetrics(definition, context) {
+	const results = [];
+	for (const entry of definition.metrics ?? []) {
+		const shape = {
+			name: entry.name,
+			...entry.unit ? { unit: entry.unit } : {},
+			...entry.better ? { better: entry.better } : {}
+		};
+		try {
+			const value = await entry.run(context);
+			if (value !== void 0 && !Number.isFinite(value)) throw new Error(`Metric ${entry.name} must return a finite number or undefined; got ${String(value)}`);
+			results.push({
+				...shape,
+				...value === void 0 ? {} : { value }
+			});
+		} catch (error) {
+			results.push({
+				...shape,
+				error: recordError(error)
+			});
+		}
+	}
+	return results;
+}
+//#endregion
+//#region ../runner/src/local-report-store.ts
+const MANIFEST_FILE = "manifest.json";
+/** The agent's session, ATXF 0.1 (RFD0009). */
+const TRANSCRIPT_FILE = "transcript.atxf.jsonl";
+/** EvalKit's events, each placed in the transcript by `at`. */
+const RUN_EVENTS_FILE = "run-events.jsonl";
+/** Sub-agent transcripts, one file each, linked from the agent's. */
+const SUBAGENTS_DIR = "transcripts";
+/** A sub-agent transcript's file name: its ID, made safe for any file system. */
+function subagentFile(id) {
+	return `${encodeURIComponent(id).replace(/\./g, "%2E")}.atxf.jsonl`;
+}
+/** The project's evals and suites when the run started. */
+const CATALOG_FILE = "catalog.json";
+async function writeJson(filePath, value) {
+	await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+async function replaceJson(filePath, value) {
+	const temporary = `${filePath}.${randomUUID()}.tmp`;
+	await writeJson(temporary, value);
+	await rename(temporary, filePath);
+}
+function assertPathSegment(value, name) {
+	if (!value || value === "." || value === ".." || value.includes("/") || value.includes("\\")) throw new Error(`${name} must be a single relative path segment`);
+}
+function artifactPath(root, value) {
+	if (!value || path.isAbsolute(value)) throw new Error(`Artifact path must be a non-empty relative path: ${value}`);
+	const normalized = path.normalize(value);
+	if (normalized === ".." || normalized.startsWith(`..${path.sep}`)) throw new Error(`Artifact path escapes its root: ${value}`);
+	return path.join(root, normalized);
+}
+var LocalRunWriter = class {
+	location;
+	directory;
+	metadata;
+	constructor(location, directory, metadata) {
+		this.location = location;
+		this.directory = directory;
+		this.metadata = metadata;
+	}
+	appendEvent(event) {
+		return appendFile(path.join(this.directory, RUN_EVENTS_FILE), `${JSON.stringify(Schema.encodeSync(RunEventSchema)(event))}\n`);
+	}
+	appendTranscript(line) {
+		return appendFile(path.join(this.directory, TRANSCRIPT_FILE), `${JSON.stringify(line)}\n`);
+	}
+	/** Written whole, to a temporary file first, so a reader never sees half of one. */
+	async writeSubagent(id, text) {
+		const directory = path.join(this.directory, SUBAGENTS_DIR);
+		await mkdir(directory, { recursive: true });
+		const file = path.join(directory, subagentFile(id));
+		const temporary = `${file}.${randomUUID()}.tmp`;
+		await writeFile(temporary, text);
+		await rename(temporary, file);
+	}
+	writeScores(scoring) {
+		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(RunScoringSchema)(scoring));
+	}
+	async writeArtifact(relativePath, data) {
+		const target = artifactPath(path.join(this.directory, "artifacts"), relativePath);
+		await mkdir(path.dirname(target), { recursive: true });
+		await writeFile(target, data);
+	}
+	async finalize(summary) {
+		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(RunSummarySchema)(summary));
+		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
+			...Schema.encodeSync(RunMetadataSchema)(this.metadata),
+			status: summary.status
+		});
+	}
+};
+/** Persists each run as a local tree rooted below `rootDirectory`. */
+function localReportStore(rootDirectory) {
+	return { async startRun(metadata, context) {
+		assertPathSegment(metadata.runId, "runId");
+		const directory = path.resolve(rootDirectory, metadata.runId);
+		await mkdir(directory, { recursive: true });
+		if (context?.catalog) await writeJson(path.join(directory, CATALOG_FILE), context.catalog);
+		await writeJson(path.join(directory, MANIFEST_FILE), {
+			...Schema.encodeSync(RunMetadataSchema)(metadata),
+			status: "running"
+		});
+		return new LocalRunWriter(path.relative(process.cwd(), directory), directory, metadata);
+	} };
+}
+//#endregion
+//#region ../core/src/run-events.ts
+/** Decodes a run's `run-events.jsonl`; a malformed line is an error naming its line number. */
+function parseRunEventsJsonl(contents) {
+	const events = [];
+	for (const [index, line] of contents.split(/\r?\n/).entries()) {
+		if (!line.trim()) continue;
+		try {
+			events.push(Schema.decodeUnknownSync(RunEventSchema)(JSON.parse(line)));
+		} catch (cause) {
+			throw new Error(`Invalid run event at line ${index + 1}`, { cause });
+		}
+	}
+	return events;
+}
+//#endregion
+//#region ../runner/src/local-report-reader.ts
+/** The local report boundary for CLI, dashboard, and CI consumers (schema version 7). */
+async function decode(file, schema) {
+	return Schema.decodeUnknownSync(schema)(JSON.parse(await readFile(file, "utf8")));
+}
+const readRunManifest = (root, runId) => decode(join(root, runId, "manifest.json"), RunManifestSchema);
+const readRunSummary = (root, runId) => decode(join(root, runId, "summary.json"), RunSummarySchema);
+const readRunScoring = (root, runId) => decode(join(root, runId, "scoring.json"), RunScoringSchema);
+/** EvalKit's events for a run, in order. */
+async function readRunEvents(root, runId) {
+	return parseRunEventsJsonl(await readFile(join(root, runId, RUN_EVENTS_FILE), "utf8"));
+}
+/** A run's sub-agent transcripts, by transcript ID; none when it kept none. */
+async function readRunSubagents(root, runId) {
+	const directory = join(root, runId, SUBAGENTS_DIR);
+	let files;
+	try {
+		files = await readdir(directory);
+	} catch {
+		return {};
+	}
+	const subagents = {};
+	for (const file of files.filter((name) => name.endsWith(".atxf.jsonl"))) {
+		const transcript = parseTranscript(await readFile(join(directory, file), "utf8"));
+		subagents[transcript.header.id] = transcript;
+	}
+	return subagents;
+}
+/**
+* The agent's ATXF transcript for a run. A running run's last line may be
+* half written; it is left out until it is complete.
+*/
+async function readRunTranscript(root, runId) {
+	const text = await readFile(join(root, runId, TRANSCRIPT_FILE), "utf8");
+	return parseTranscript(text.slice(0, text.lastIndexOf("\n") + 1));
+}
+/**
+* True when a run's owning process is known to be gone: it ran on this host
+* and its PID no longer exists. Runs from other hosts are never judged.
+*/
+function ownerIsGone(owner) {
+	if (!owner || owner.hostname !== os.hostname()) return false;
+	if (owner.pid === process.pid) return false;
+	try {
+		process.kill(owner.pid, 0);
+		return false;
+	} catch (error) {
+		return error.code === "ESRCH";
+	}
+}
+//#endregion
+//#region ../runner/src/matrix.ts
+/** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
+function runMatrix(matrix, options) {
+	return Effect.gen(function* () {
+		const concurrency = options.concurrency ?? 4;
+		if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Concurrency must be a positive integer");
+		const count = matrix.count(options.selection);
+		const iterator = matrix.cells(options.selection);
+		const summary = {
+			cells: 0,
+			runs: 0,
+			passed: 0,
+			failed: 0,
+			limited: 0,
+			cancelled: 0
+		};
+		yield* Effect.all(Array.from({ length: Math.min(count, concurrency) }, () => Effect.gen(function* () {
+			while (!options.signal?.aborted) {
+				const next = iterator.next();
+				if (next.done) return;
+				const cell = next.value;
+				const runtime = options.runtime ?? [
+					"local",
+					"remote",
+					"sandbox"
+				].find((name) => agentRuntimes(cell.eval.agent)[name]);
+				const suiteId = options.suiteId ?? options.suiteFor?.(cell);
+				const runs = yield* runEval(cell.eval, {
+					...options,
+					...suiteId ? { suiteId } : {},
+					concurrency: 1,
+					runtime,
+					parameters: cell.parameters,
+					matrix: {
+						id: matrix.id,
+						cellKey: cell.key
+					}
+				});
+				summary.cells++;
+				for (const run of runs) {
+					summary.runs++;
+					if (run.status === "cancelled") summary.cancelled++;
+					else if (run.status === "passed") summary.passed++;
+					else if (run.status === "limited") summary.limited++;
+					else summary.failed++;
+				}
+				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, runs));
+			}
+		})), { concurrency });
+		return summary;
+	});
+}
+//#endregion
+//#region ../runner/src/upgrade.ts
+/** A new run being written. */
+const TEMPORARY = ".upgrade-";
+/** An old run set aside while its replacement, with the same ID, moves in. */
+const RETIRED = ".upgraded-";
+const PER_RUN_FILES = ["archived.json"];
+/**
+* Rewrites results from report schema versions 2 to 5 into version 6, in
+* place (RFD0008). An old run of one trial (most of them) becomes a run with
+* the same ID, so links to it keep working. An old run of several trials
+* becomes runs with the trials' IDs, in a batch with the old run's ID.
+*
+* Each new run is written under a temporary name and renamed into place;
+* artifacts are moved, not copied, as the last step before that rename. An
+* old run is removed only after its new runs exist, and one replaced under
+* the same ID is set aside first, so a crash at any point leaves the old run
+* or complete new ones, and re-running finishes the job. Version 6 runs are
+* left alone. Then every version 6 run is converted to version 7, its
+* trajectory into an ATXF transcript (RFD0009).
+*/
+async function upgradeReports(root, options = {}) {
+	const report = {
+		upgraded: 0,
+		runs: 0,
+		batches: 0,
+		running: [],
+		transcripts: 0,
+		failed: []
+	};
+	let entries;
+	try {
+		entries = await readdir(root);
+	} catch {
+		return report;
+	}
+	if (!options.dryRun) await finishReplacements(root, entries);
+	for (const entry of entries) {
+		if (entry.startsWith(".")) continue;
+		const directory = join(root, entry);
+		const manifest = await readJson(join(directory, "manifest.json"));
+		if (!manifest || !UPGRADABLE_SCHEMA_VERSIONS.includes(manifest.schemaVersion)) continue;
+		const summary = await readJson(join(directory, "summary.json"));
+		if (!summary && !ownerIsGone(manifest.process)) {
+			report.running.push(entry);
+			continue;
+		}
+		const trials = await trialIds(directory);
+		report.upgraded += 1;
+		report.runs += Math.max(trials.length, 1);
+		if (trials.length > 1) report.batches += 1;
+		if (options.dryRun) {
+			report.transcripts += Math.max(trials.length, 1);
+			continue;
+		}
+		await upgradeRun(root, entry, manifest, summary, trials);
+	}
+	if (!options.dryRun) await removeLeftovers(root);
+	for (const entry of await readdir(root)) {
+		if (entry.startsWith(".")) continue;
+		const directory = join(root, entry);
+		const manifest = await readJson(join(directory, "manifest.json"));
+		if (manifest?.schemaVersion !== 6) continue;
+		if (!await readJson(join(directory, "summary.json")) && !ownerIsGone(manifest.process)) {
+			if (!report.running.includes(entry)) report.running.push(entry);
+			continue;
+		}
+		if (options.dryRun) {
+			report.transcripts += 1;
+			continue;
+		}
+		try {
+			await transcriptRun(directory, entry);
+			report.transcripts += 1;
+		} catch (error) {
+			report.failed.push({
+				runId: entry,
+				reason: error instanceof Error ? error.message : String(error)
+			});
+		}
+	}
+	return report;
+}
+/** Where a version 6 run's own files wait until its version 7 files are in place. */
+const V6_BACKUP = ".v6";
+const V6_FILES = [
+	"manifest.json",
+	"trajectory.jsonl",
+	"scoring.json",
+	"summary.json"
+];
+/**
+* Converts one version 6 run to version 7 in place (RFD0009): its
+* trajectory becomes an ATXF transcript, sealed, plus `run-events.jsonl`;
+* its scoring and summary point into the transcript. The old files are
+* copied to `.v6/` first and read from there, so a conversion a crash
+* interrupted starts again from them; the manifest's version changes last.
+*/
+async function transcriptRun(directory, runId) {
+	const backup = join(directory, V6_BACKUP);
+	if (!await exists(backup)) {
+		await mkdir(`${backup}.tmp`, { recursive: true });
+		for (const file of V6_FILES) if (await exists(join(directory, file))) await cp(join(directory, file), join(`${backup}.tmp`, file));
+		await rename(`${backup}.tmp`, backup);
+	}
+	const manifest = await readJson(join(backup, "manifest.json"));
+	const events = (await readText(join(backup, "trajectory.jsonl")) ?? "").split("\n").filter((line) => line.trim()).map((line, index) => {
+		try {
+			return JSON.parse(line);
+		} catch {
+			throw new Error(`trajectory.jsonl line ${index + 1} is not JSON`);
+		}
+	});
+	const startedAt = typeof manifest.startedAt === "string" ? manifest.startedAt : (/* @__PURE__ */ new Date(0)).toISOString();
+	const aut = manifest.aut;
+	const converted = convertTrajectory(events, {
+		runId,
+		...typeof manifest.evalId === "string" ? { evalId: manifest.evalId } : {},
+		startedAt,
+		...aut ? { agent: aut } : {}
+	});
+	const lastTime = converted.transcript.records.at(-1)?.time ?? converted.transcript.header.time;
+	const transcript = addIntegrity(converted.transcript, { seal: {
+		id: "seal",
+		time: lastTime
+	} });
+	const text = serializeTranscript(transcript);
+	const invalid = validateTranscript(text).diagnostics.filter((d) => d.severity === "error");
+	if (invalid.length) throw new Error(`its converted transcript is not valid ATXF: ${invalid.slice(0, 3).map((d) => `line ${d.line}: ${d.message}`).join("; ")}`);
+	const decode = (schema, value, file) => {
+		try {
+			return Schema.decodeUnknownSync(schema)(value);
+		} catch (error) {
+			const issue = ParseResult.isParseError(error) ? ParseResult.ArrayFormatter.formatErrorSync(error)[0] : void 0;
+			throw new Error(issue ? `its converted ${file} is not valid at ${issue.path.join(".") || "(root)"}: ${issue.message}` : `its converted ${file} is not valid: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	};
+	const scoringIn = await readJson(join(backup, "scoring.json"));
+	const scoring = scoringIn ? decode(RunScoringSchema, upgradeScoring(scoringIn, {
+		runId,
+		startedAt
+	}, converted.recordOfEvent, converted.transcript), "scoring.json") : void 0;
+	const summaryIn = await readJson(join(backup, "summary.json"));
+	const summary = summaryIn ? decode(RunSummarySchema, {
+		...summaryIn,
+		...isObject(summaryIn.scoring) ? { scoring: upgradeScoring(summaryIn.scoring, {
+			runId,
+			startedAt
+		}, converted.recordOfEvent, converted.transcript) } : {}
+	}, "summary.json") : void 0;
+	await writeFile(join(directory, TRANSCRIPT_FILE), text);
+	await writeFile(join(directory, RUN_EVENTS_FILE), converted.runEvents.map((event) => `${JSON.stringify(event)}\n`).join(""));
+	if (scoring) await writeJsonFile(join(directory, "scoring.json"), scoring);
+	if (summary) await writeJsonFile(join(directory, "summary.json"), summary);
+	await writeJsonFile(join(directory, "manifest.json"), {
+		...manifest,
+		schemaVersion: 7
+	});
+	await rm(join(directory, "trajectory.jsonl"), { force: true });
+	await rm(backup, {
+		recursive: true,
+		force: true
+	});
+}
+const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/**
+* Finishes same-ID replacements a crash interrupted: the old run was set
+* aside, and its replacement may or may not have moved in.
+*/
+async function finishReplacements(root, entries) {
+	for (const entry of entries) {
+		if (!entry.startsWith(RETIRED)) continue;
+		const id = entry.slice(10);
+		const retired = join(root, entry);
+		const directory = join(root, id);
+		const replacement = join(root, `${TEMPORARY}${id}`);
+		if (!await exists(directory)) await rename(await exists(replacement) ? replacement : retired, directory);
+		await rm(retired, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/**
+* Temporary folders nothing will finish. One holding artifacts is kept: it
+* could be the only copy, and an old run would have moved them back.
+*/
+async function removeLeftovers(root) {
+	for (const entry of await readdir(root)) if (entry.startsWith(TEMPORARY) && !await exists(join(root, entry, "artifacts"))) await rm(join(root, entry), {
+		recursive: true,
+		force: true
+	});
+}
+/** Swaps a fully written replacement in for the old run of the same ID. */
+async function replaceInPlace(root, id, replacement) {
+	const retired = join(root, `${RETIRED}${id}`);
+	await rename(join(root, id), retired);
+	await rename(replacement, join(root, id));
+	await rm(retired, {
+		recursive: true,
+		force: true
+	});
+}
+async function upgradeRun(root, oldRunId, runManifest, runSummary, trials) {
+	const directory = join(root, oldRunId);
+	const catalog = await readJson(join(directory, "catalog.json"));
+	const extras = await presentFiles(directory, PER_RUN_FILES);
+	if (!trials.length) {
+		const target = join(root, `${TEMPORARY}${oldRunId}`);
+		await mkdir(target, { recursive: true });
+		const status = runSummary ? "cancelled" : "running";
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			runId: oldRunId,
+			status
+		}));
+		if (runSummary) await writeJsonFile(join(target, "summary.json"), {
+			status,
+			endedAt: runSummary.endedAt,
+			...runSummary.durationMs === void 0 ? {} : { durationMs: runSummary.durationMs },
+			...runSummary.error ? { error: runSummary.error } : {}
+		});
+		await copyShared(directory, target, catalog, extras);
+		await replaceInPlace(root, oldRunId, target);
+		return;
+	}
+	const inPlace = trials.length === 1;
+	for (const trialId of trials) {
+		const runId = inPlace ? oldRunId : trialId;
+		const destination = join(root, runId);
+		if (!inPlace && await exists(join(destination, "manifest.json"))) continue;
+		const source = join(directory, "trials", trialId);
+		const trialManifest = await readJson(join(source, "manifest.json")) ?? {};
+		const trialSummary = await readJson(join(source, "summary.json"));
+		const passed = (trialSummary?.scoring)?.passed;
+		const status = upgradeStatus(trialSummary?.status ?? trialManifest.status, passed);
+		const index = typeof trialManifest.trialIndex === "number" ? trialManifest.trialIndex : trials.indexOf(trialId);
+		const target = join(root, `${TEMPORARY}${runId}`);
+		if (!await exists(join(source, "artifacts")) && await exists(join(target, "artifacts"))) await rename(join(target, "artifacts"), join(source, "artifacts"));
+		await rm(target, {
+			recursive: true,
+			force: true
+		});
+		await mkdir(target, { recursive: true });
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			trial: trialManifest,
+			runId,
+			...!inPlace ? { batch: {
+				id: oldRunId,
+				index,
+				size: trials.length
+			} } : {},
+			status
+		}));
+		if (trialSummary) await writeJsonFile(join(target, "summary.json"), {
+			...trialSummary,
+			status
+		});
+		const scoring = await readText(join(source, "scoring.json"));
+		if (scoring !== void 0) await writeFile(join(target, "scoring.json"), scoring);
+		const trajectory = await readText(join(source, "trajectory.jsonl"));
+		if (trajectory !== void 0) await writeFile(join(target, "trajectory.jsonl"), upgradeTrajectory(trajectory));
+		await copyShared(directory, target, catalog, extras);
+		if (await exists(join(source, "artifacts"))) await rename(join(source, "artifacts"), join(target, "artifacts"));
+		if (inPlace) await replaceInPlace(root, oldRunId, target);
+		else await rename(target, destination);
+	}
+	if (!inPlace) await rm(directory, {
+		recursive: true,
+		force: true
+	});
+}
+async function copyShared(from, to, catalog, extras) {
+	if (catalog) await writeJsonFile(join(to, "catalog.json"), upgradeCatalog(catalog));
+	for (const file of extras) await cp(join(from, file), join(to, file));
+}
+async function trialIds(directory) {
+	try {
+		return (await readdir(join(directory, "trials"), { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
+	} catch {
+		return [];
+	}
+}
+async function presentFiles(directory, names) {
+	const present = [];
+	for (const name of names) if (await exists(join(directory, name))) present.push(name);
+	return present;
+}
+async function exists(path) {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function readText(path) {
+	try {
+		return await readFile(path, "utf8");
+	} catch {
+		return;
+	}
+}
+async function readJson(path) {
+	const text = await readText(path);
+	if (text === void 0) return void 0;
+	let value;
+	try {
+		value = JSON.parse(text);
+	} catch (error) {
+		throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+async function writeJsonFile(path, value) {
+	await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+//#endregion
+//#region ../runner/src/index.ts
+/**
+* Runs a scenario: once, or `repeat` times as one batch. Resolves to one
+* result per run started; runs never started because of cancellation are
+* left out.
+*/
+function runEval(definition, options) {
+	return Effect.gen(function* () {
+		yield* Effect.logInfo("eval run started").pipe(Effect.annotateLogs({
+			evalId: authoringId(definition),
+			...options.runId ? { runId: options.runId } : {},
+			...options.suiteId ? { suiteId: options.suiteId } : {}
+		}));
+		const runs = yield* Effect.tryPromise({
+			try: () => executeBatch(definition, options),
+			catch: (error) => error
+		});
+		yield* Effect.logInfo("eval run finished").pipe(Effect.annotateLogs({
+			evalId: authoringId(definition),
+			runs: runs.length,
+			passed: runs.filter((run) => run.status === "passed").length
+		}));
+		return runs;
+	});
+}
+function createId() {
+	return crypto.randomUUID();
+}
+function canonicalId(kind, id) {
+	return resourceUri(kind, id);
+}
+function assertUuid(value, label) {
+	if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error(`${label} must be a UUID`);
+}
+function runMetadata(definition, options, runId, startedAt, extra) {
+	return {
+		schemaVersion: 7,
+		runId,
+		runUri: canonicalId("run", runId),
+		evalId: definition.id,
+		...options.suiteId ? { suiteId: options.suiteId } : {},
+		...options.source ? { source: options.source } : {},
+		process: {
+			pid: process.pid,
+			hostname: os.hostname()
+		},
+		...definition.agent.identity ? { aut: definition.agent.identity } : {},
+		parameters: options.parameters,
+		matrix: options.matrix,
+		startedAt,
+		...options.batch ? { batch: options.batch } : {},
+		...extra
+	};
+}
+/** Recording a secondary error must never mask the original failure or block cleanup. */
+async function tryEmitError(error, emitRunner, now) {
+	try {
+		await emitRunner({
+			kind: "error",
+			error: recordError(error),
+			timestamp: now().toISOString()
+		});
+	} catch {}
+}
+/** Starts `repeat` runs of a scenario, as one batch when there are several. */
+async function executeBatch(definition, options) {
+	let hasUser = false;
+	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
+	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
+	assertNoRemovedOptions(definition);
+	assertJudgesIndependent(definition, options.judge);
+	for (const rule of [...definition.transcript, ...definition.scoring]) if (rule.kind === "judge" && !rule.agent && !options.judge) throw new Error(`Judge ${rule.name} in ${definition.id} has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts`);
+	if (options.runId) assertUuid(options.runId, "runId");
+	const repeat = options.repeat ?? definition.repeat ?? 1;
+	if (!Number.isInteger(repeat) || repeat < 1) throw new Error(`repeat must be a positive integer; received ${repeat}`);
+	if (repeat === 1) return [await runOnce(definition, options)];
+	if (options.runId) throw new Error("runId names one run; it cannot be used with repeat");
+	const batchId = createId();
+	const signal = options.signal;
+	const runs = Array.from({ length: repeat }, (_, index) => {
+		const run = Effect.tryPromise({
+			try: async () => signal?.aborted ? void 0 : runOnce(definition, {
+				...options,
+				batch: {
+					id: batchId,
+					index,
+					size: repeat
+				}
+			}),
+			catch: (error) => error
+		});
+		return options.semaphore ? options.semaphore.withPermits(1)(run) : run;
+	});
+	return (await Effect.runPromise(Effect.all(runs, { concurrency: options.concurrency ?? 32 }))).filter((run) => run !== void 0);
+}
+/** One execution: a session in its own sandbox, scored and reported. */
+async function runOnce(definition, options) {
+	const now = options.now ?? (() => /* @__PURE__ */ new Date());
+	const runId = options.runId ?? createId();
+	const startedAt = now().toISOString();
+	const selectedRuntime = options.runtime;
+	const runtime = selectedRuntime ? agentRuntimes(definition.agent)[selectedRuntime] : void 0;
+	if (selectedRuntime && !runtime) throw new Error(`AUT does not declare the requested ${selectedRuntime} runtime`);
+	const randomSeed = crypto.getRandomValues(/* @__PURE__ */ new Uint32Array(1))[0];
+	const batch = options.batch;
+	const signal = options.signal ?? new AbortController().signal;
+	const root = options.root ?? process.cwd();
+	const baseContext = {
+		runId,
+		evalId: authoringId(definition),
+		...batch ? { batch } : {},
+		metadata: {
+			...toJsonObject(definition.metadata ?? {}, `${definition.id}.metadata`),
+			randomSeed
+		},
+		...options.parameters ? { parameters: options.parameters } : {},
+		...selectedRuntime ? { runtime: selectedRuntime } : {}
+	};
+	let datasetsError;
+	let prepared = [];
+	try {
+		prepared = await prepareDatasets(definition.datasets ?? [], root, {
+			evalId: baseContext.evalId,
+			runId,
+			...batch ? { batch } : {},
+			seed: randomSeed,
+			parameters: options.parameters ?? {},
+			signal
+		});
+	} catch (error) {
+		datasetsError = error;
+	}
+	const candidateData = datasetValues(prepared, "candidate");
+	const scoringData = datasetValues(prepared, "all");
+	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
+	const limits = resolveLimits(options.limits?.base ?? {}, parseLimits(definition.limits, `Eval ${definition.id}`), limitsFromParameters(options.parameters), options.limits?.override ?? {});
+	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, startedAt, {
+		workspaceIgnore,
+		...prepared.length ? { datasets: runDatasets(prepared) } : {},
+		...Object.keys(limits).length ? { limits } : {},
+		...options.isolation?.kind === "container" ? { isolation: {
+			kind: "container",
+			image: options.isolation.image,
+			network: options.isolation.network
+		} } : options.isolation?.kind === "sandbox-exec" ? { isolation: { kind: "sandbox-exec" } } : {}
+	}), { catalog: options.catalog });
+	const attachments = attachmentStore(runWriter, options.attachmentLimits);
+	let writeChain = Promise.resolve();
+	let reportingFailed = false;
+	const runRef = {
+		runId,
+		...batch ? { batch } : {},
+		evalId: definition.id,
+		...options.suiteId ? { suiteId: options.suiteId } : {},
+		...options.parameters ? { parameters: options.parameters } : {}
+	};
+	const observe = (update) => {
+		try {
+			options.onUpdate?.(runRef, update);
+		} catch {}
+	};
+	const limitController = new AbortController();
+	const runSignal = AbortSignal.any([signal, limitController.signal]);
+	let reached;
+	let steps = 0;
+	let tokens = 0;
+	const reach = (limit) => {
+		if (reached || signal.aborted) return;
+		reached = limit;
+		emitRunner({
+			kind: "limit-reached",
+			...limit,
+			timestamp: now().toISOString()
+		}).catch(() => {});
+		limitController.abort(new LimitReachedError({
+			message: `Stopped at its ${limit.limit} limit (${formatLimit(limit.limit, limit.value)})`,
+			limit
+		}));
+	};
+	const countRecord = (record) => {
+		if (record.type === "message" && record.role === "assistant") {
+			steps += 1;
+			if (record.usage) tokens += uncachedTokens(record.usage);
+		} else if (record.type === "usage" && !record.cumulative || record.type === "compaction" || record.type === "branch_summary") tokens += record.usage ? uncachedTokens(record.usage) : 0;
+		if (limits.steps !== void 0 && steps > limits.steps) reach({
+			limit: "steps",
+			value: limits.steps,
+			used: steps
+		});
+		else if (limits.tokens !== void 0 && tokens > limits.tokens) reach({
+			limit: "tokens",
+			value: limits.tokens,
+			used: tokens
+		});
+	};
+	let timeLimit;
+	let sessionStarted;
+	const transcript = new RunTranscript({
+		id: runId,
+		agent: {
+			id: definition.agent.identity?.id ?? "agent",
+			...definition.agent.identity?.version ? { version: definition.agent.identity.version } : {}
+		},
+		runId,
+		scenarioId: definition.id,
+		now,
+		persist: (line) => runWriter.appendTranscript(line),
+		persistSubagent: (id, text) => runWriter.writeSubagent(id, text),
+		onRecord: (record) => {
+			countRecord(record);
+			observe({
+				kind: "record",
+				record
+			});
+		},
+		onSubagentRecord: countRecord
+	});
+	const runEvents = [];
+	const emitRunner = async (input) => {
+		const event = {
+			...input,
+			at: transcript.leafId
+		};
+		runEvents.push(event);
+		writeChain = writeChain.then(() => runWriter.appendEvent(event));
+		try {
+			await writeChain;
+			observe({
+				kind: "event",
+				event
+			});
+		} catch (cause) {
+			reportingFailed = true;
+			throw new ReportError({
+				cause,
+				message: cause instanceof Error ? cause.message : String(cause)
+			});
+		}
+	};
+	let workspace;
+	let runShell;
+	let context;
+	let primaryError;
+	let closeError;
+	let scoring;
+	let artifacts;
+	let status = "running";
+	const progress = {
+		checkpoints: [],
+		stoppedEarly: false
+	};
+	let cleanupError;
+	const workspaceScope = await Effect.runPromise(Scope.make());
+	try {
+		await emitRunner({
+			kind: "run-started",
+			timestamp: now().toISOString()
+		});
+		if (signal.aborted) throw cancelled();
+		if (datasetsError) throw new DatasetError({
+			cause: datasetsError,
+			message: datasetsError instanceof Error ? datasetsError.message : String(datasetsError)
+		});
+		workspace = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
+			try: () => createRunWorkspace(runId, prepared, options.workspaceRoot),
+			catch: (cause) => new DatasetError({
+				cause,
+				message: cause instanceof Error ? cause.message : String(cause)
+			})
+		}), (created) => Effect.tryPromise({
+			try: () => created.cleanup(keepWorkspace(options.keepWorkspaces, scoring?.passed === true)),
+			catch: (cause) => new DatasetError({
+				cause,
+				message: cause instanceof Error ? cause.message : String(cause)
+			})
+		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
+			cleanupError = error;
+		}).pipe(Effect.tap(() => Effect.logError("run workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
+		const runWorkspace = workspace;
+		runShell = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
+			try: () => openRunShell({
+				isolation: options.isolation ?? none(),
+				runId,
+				root: runWorkspace.artifacts.candidate.root,
+				projectRoot: options.root ?? process.cwd(),
+				signal: runSignal,
+				...options.fixtures ? { fixtures: options.fixtures } : {}
+			}),
+			catch: (cause) => new AutExecutionError({
+				cause,
+				message: `Could not set up the run's ${isolationLabel(options.isolation)}: ${cause instanceof Error ? cause.message : String(cause)}`
+			})
+		}), (opened) => Effect.promise(() => opened.close({ keep: keepWorkspace(options.keepWorkspaces, scoring?.passed === true) }).catch((error) => {
+			cleanupError ??= error;
+		}))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
+		const runContext = {
+			...baseContext,
+			signal: runSignal,
+			limits,
+			shell: runShell.shell,
+			fixtures: runShell.fixtures,
+			data: candidateData,
+			workspace: runWorkspace.artifacts.candidate,
+			evaluatorWorkspace: runWorkspace.artifacts.evaluator
+		};
+		context = runContext;
+		sessionStarted = Date.now();
+		if (limits.time !== void 0) {
+			const time = limits.time;
+			timeLimit = setTimeout(() => reach({
+				limit: "time",
+				value: time,
+				used: Date.now() - (sessionStarted ?? Date.now())
+			}), time);
+			timeLimit.unref?.();
+		}
+		await runBoundary(Effect.acquireUseRelease(Effect.tryPromise({
+			try: () => definition.agent.start({
+				context: runContext,
+				...runtime ? { runtime } : {},
+				transcript: transcript.writer(),
+				attach: attachments.attach
+			}),
+			catch: (cause) => new AutExecutionError({
+				cause,
+				message: cause instanceof Error ? cause.message : String(cause)
+			})
+		}), (session) => Effect.tryPromise({
+			try: () => executeTranscript(definition, session, {
+				context: runContext,
+				artifacts: runWorkspace.artifacts,
+				transcript,
+				runEvents,
+				now,
+				randomSeed,
+				emitRunner,
+				data: scoringData,
+				signal: runSignal,
+				progress,
+				attach: attachments.attach,
+				defaultJudge: options.judge
+			}),
+			catch: (cause) => cause
+		}), (opened) => Effect.tryPromise({
+			try: async () => opened.close?.(),
+			catch: (cause) => new AutExecutionError({
+				cause,
+				message: cause instanceof Error ? cause.message : String(cause)
+			})
+		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
+			closeError = error;
+		}).pipe(Effect.tap(() => Effect.logError("AUT session close failed")))))).pipe(Effect.annotateLogs({
+			runId,
+			evalId: definition.id
+		})));
+	} catch (error) {
+		primaryError = error;
+		if (!(error instanceof LimitReachedError)) await tryEmitError(error, emitRunner, now);
+	}
+	if (timeLimit) clearTimeout(timeLimit);
+	if (closeError) {
+		if (!primaryError) primaryError = closeError;
+		await tryEmitError(closeError, emitRunner, now);
+	}
+	transcript.seal();
+	const problems = transcript.problems();
+	if (problems.length) {
+		const invalid = new AutExecutionError({
+			cause: problems,
+			message: `The agent's transcript is not valid ATXF: ${problems.slice(0, 3).join("; ")}${problems.length > 3 ? ` (${problems.length - 3} more)` : ""}`
+		});
+		if (!primaryError) primaryError = invalid;
+		await tryEmitError(invalid, emitRunner, now);
+	}
+	if (reached && primaryError === void 0) primaryError = limitController.signal.reason;
+	const limited = primaryError instanceof LimitReachedError;
+	if (limited) primaryError = void 0;
+	const wasCancelled = primaryError instanceof CancelledError;
+	const { checkpoints, stoppedEarly, lastTurn } = progress;
+	const scoreResults = [];
+	const skippedScorers = [];
+	const metrics = [];
+	try {
+		await evaluateFinalScorers(definition, {
+			context,
+			workspace,
+			agentTranscript: transcript,
+			runEvents,
+			lastTurn,
+			primaryError,
+			limited,
+			stoppedEarly,
+			cancelled: wasCancelled,
+			reportingFailed,
+			results: scoreResults,
+			skippedScorers,
+			data: scoringData,
+			checkpoints,
+			attach: attachments.attach,
+			defaultJudge: options.judge,
+			now,
+			emitRunner
+		});
+		metrics.push(...builtInMetrics(transcript.transcript, runEvents, [...scoreResults, ...checkpoints], Object.values(transcript.subagents)));
+		if (context && workspace && !wasCancelled) metrics.push(...await evaluateMetrics(definition, scoringContext({
+			context,
+			workspace,
+			agentTranscript: transcript,
+			runEvents,
+			lastTurn,
+			data: scoringData,
+			checkpoints,
+			transcript: definition.transcript,
+			attach: attachments.attach
+		})));
+		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
+		await runWriter.writeScores(scoring);
+		if (workspace) try {
+			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, runWriter, workspaceIgnore);
+		} catch (captureError) {
+			if (!primaryError) primaryError = captureError;
+			await tryEmitError(captureError, emitRunner, now);
+		}
+	} catch (error) {
+		if (!primaryError) primaryError = error;
+		await tryEmitError(error, emitRunner, now);
+	} finally {
+		await Effect.runPromise(Scope.close(workspaceScope, Exit.succeed(void 0)));
+		if (cleanupError) {
+			if (!primaryError) primaryError = cleanupError;
+			await tryEmitError(cleanupError, emitRunner, now);
+		}
+	}
+	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
+	if (!primaryError) try {
+		await emitRunner({
+			kind: "run-completed",
+			timestamp: now().toISOString()
+		});
+	} catch (error) {
+		primaryError = error;
+	}
+	try {
+		await transcript.settled();
+	} catch (cause) {
+		primaryError ??= new ReportError({
+			cause,
+			message: cause instanceof Error ? cause.message : String(cause)
+		});
+	}
+	status = wasCancelled ? "cancelled" : primaryError ? "errored" : limited ? "limited" : scoring.passed ? "passed" : "failed";
+	if (options.fixtures?.started) for (const log of options.fixtures.logs()) await attachments.attach(`logs/fixtures/${log.name}.log`, log.content).catch(() => void 0);
+	const error = primaryError ? recordError(primaryError) : void 0;
+	const attached = await attachments.close();
+	const endedAt = now().toISOString();
+	const durationMs = new Date(endedAt).getTime() - new Date(startedAt).getTime();
+	await runWriter.finalize({
+		status,
+		endedAt,
+		durationMs,
+		scoring,
+		...artifacts ? { artifacts } : {},
+		...attached.length ? { attachments: attached } : {},
+		...error ? { error } : {},
+		...limited && reached ? { limit: reached } : {}
+	});
+	return {
+		runId,
+		runUri: canonicalId("run", runId),
+		evalId: definition.id,
+		status,
+		...batch ? { batch } : {},
+		reportLocation: runWriter.location,
+		durationMs,
+		scoring,
+		...error ? { error } : {},
+		...limited && reached ? { limit: reached } : {}
+	};
+}
+function datasetValues(prepared, audience) {
+	const values = {};
+	for (const { dataset, value } of prepared) {
+		if (value === void 0) continue;
+		if (audience === "candidate" && dataset.visibility !== "candidate") continue;
+		values[dataset.id] = value;
+	}
+	return values;
+}
+/** Persistent workspaces of runs that did not pass are kept for debugging by default. */
+function isolationLabel(isolation) {
+	if (isolation?.kind === "container") return "container";
+	if (isolation?.kind === "sandbox-exec") return "sandbox";
+	return "shell";
+}
+function keepWorkspace(policy = "failed", passed) {
+	return policy === "all" || policy === "failed" && !passed;
+}
+//#endregion
+export { ScoringError as C, ReportError as S, removeRunContainer as _, readRunEvents as a, CheckpointExecutionError as b, readRunSubagents as c, localReportStore as d, runWorkspacePath as f, fixtureSet as g, globMatcher as h, ownerIsGone as i, readRunSummary as l, staticDatasetFiles as m, upgradeReports as n, readRunManifest as o, staticDatasetFile as p, runMatrix as r, readRunScoring as s, runEval as t, readRunTranscript as u, AutExecutionError as v, DatasetError as x, CancelledError as y };