@leostera/evalkit 0.9.0 → 0.11.0

package/dist/src-CcdLZqFk.mjs

@@ -1,1714 +0,0 @@
-import { $ as relativeDatasetPath, A as RunManifestSchema, B as turnResultEvents, C as CheckpointResultSchema, F as RunSummarySchema, G as DEFAULT_ATTACHMENT_LIMITS, H as canonicalParameters, I as ScoreResultSchema, L as TrajectoryEventSchema, M as RunScoringSchema, U as turnTokens, W as DEFAULT_ARTIFACT_EXCLUDE, X as toJsonObject, Y as toJson, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as RunMetadataSchema, ot as resourceUri, p as recordError, w as JsonValueSchema, z as normalizeAutEvent } from "./src-BrfIXfO2.mjs";
-import { a as upgradeManifest, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS } from "./upgrade-60SIRius.mjs";
-import * as Schema from "effect/Schema";
-import { Data, Effect, Either, Exit, Scope } from "effect";
-import os from "node:os";
-import { appendFile, copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
-import path, { join } from "node:path";
-import { createHash, randomUUID } from "node:crypto";
-//#region ../runner/src/errors.ts
-var AutExecutionError = class extends Data.TaggedError("AutExecutionError") {};
-var DatasetError = class extends Data.TaggedError("DatasetError") {};
-var ReportError = class extends Data.TaggedError("ReportError") {};
-var ScoringError = class extends Data.TaggedError("ScoringError") {};
-var RuleExecutionError = class extends Data.TaggedError("RuleExecutionError") {};
-var CheckpointExecutionError = class extends Data.TaggedError("CheckpointExecutionError") {};
-/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
-var CancelledError = class extends Data.TaggedError("CancelledError") {};
-function cancelled() {
-	return new CancelledError({ message: "Cancelled" });
-}
-/** Settles like `work`, or rejects with `CancelledError` as soon as `signal` aborts. */
-async function unlessAborted(work, signal) {
-	if (signal.aborted) throw cancelled();
-	let onAbort;
-	const aborted = new Promise((_, reject) => {
-		onAbort = () => reject(cancelled());
-		signal.addEventListener("abort", onAbort, { once: true });
-	});
-	try {
-		return await Promise.race([work(), aborted]);
-	} finally {
-		signal.removeEventListener("abort", onAbort);
-	}
-}
-//#endregion
-//#region ../runner/src/attachments.ts
-const SEGMENT = /^[A-Za-z0-9_-][A-Za-z0-9._-]{0,99}$/;
-const MAX_NAME = 200;
-/**
-* Checks an attachment name: a relative path whose segments are letters,
-* digits, `.`, `_` and `-`, not starting with a dot. That rules out
-* absolute paths, `..`, hidden files, separators other than `/`, and
-* anything a filesystem or URL would read differently.
-*/
-function attachmentName(name) {
-	if (typeof name !== "string" || !name || name.length > MAX_NAME || !name.split("/").every((segment) => SEGMENT.test(segment))) throw new Error(`Attachment name ${JSON.stringify(name)} is not allowed: use a relative path of letters, digits, ".", "_" and "-", like "logs/agent.log"`);
-	return name;
-}
-/** The last `max` bytes, starting on a whole UTF-8 character when the content is text. */
-function lastBytes(bytes, max, text) {
-	let start = bytes.byteLength - max;
-	if (text) while (start < bytes.byteLength && (bytes[start] & 192) === 128) start++;
-	return bytes.subarray(start);
-}
-/**
-* Saves a run's attachments as `attachments/<name>` below its artifacts.
-* Content is stored as given and never interpreted. A file over
-* `maxFileBytes` keeps its end (where a log's errors are); one that would
-* take the run past `maxRunBytes` is rejected.
-*/
-function attachmentStore(writer, limits = {}) {
-	const { maxFileBytes, maxRunBytes } = {
-		...DEFAULT_ATTACHMENT_LIMITS,
-		...limits
-	};
-	const entries = [];
-	const names = /* @__PURE__ */ new Set();
-	const pending = /* @__PURE__ */ new Set();
-	let used = 0;
-	let closed = false;
-	async function attach(name, content) {
-		if (closed) throw new Error(`Cannot attach ${JSON.stringify(name)}: the run has finished`);
-		attachmentName(name);
-		if (typeof content !== "string" && !(content instanceof Uint8Array)) throw new Error(`Attachment ${name} must be a string or a Uint8Array, not ${content === null ? "null" : typeof content}`);
-		if (names.has(name)) throw new Error(`Attachment ${name} was already attached`);
-		const text = typeof content === "string";
-		const original = text ? new TextEncoder().encode(content) : content;
-		const bytes = original.byteLength > maxFileBytes ? lastBytes(original, maxFileBytes, text) : original;
-		if (used + bytes.byteLength > maxRunBytes) throw new Error(`Attachment ${name} (${bytes.byteLength} bytes) would take this run past its ${maxRunBytes}-byte attachment limit (${used} used)`);
-		names.add(name);
-		used += bytes.byteLength;
-		const write = writer.writeArtifact(`attachments/${name}`, bytes);
-		pending.add(write);
-		try {
-			await write;
-			entries.push({
-				name,
-				size: bytes.byteLength,
-				...bytes.byteLength < original.byteLength ? { truncatedFrom: original.byteLength } : {}
-			});
-		} catch (error) {
-			names.delete(name);
-			used -= bytes.byteLength;
-			throw error;
-		} finally {
-			pending.delete(write);
-		}
-	}
-	return {
-		attach,
-		async close() {
-			closed = true;
-			await Promise.allSettled(pending);
-			return [...entries].sort((a, b) => a.name.localeCompare(b.name));
-		}
-	};
-}
-//#endregion
-//#region ../core/src/glob.ts
-/**
-* A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
-* spans segments, and `{a,b}` is an alternation.
-*/
-function globMatcher(pattern) {
-	let source = "";
-	for (let index = 0; index < pattern.length; index++) {
-		const char = pattern[index];
-		if (char === "*" && pattern[index + 1] === "*") {
-			index++;
-			if (pattern[index + 1] === "/") {
-				index++;
-				source += "(?:.*/)?";
-			} else source += ".*";
-		} else if (char === "*") source += "[^/]*";
-		else if (char === "?") source += "[^/]";
-		else if (char === "{") {
-			const end = pattern.indexOf("}", index);
-			if (end === -1) source += "\\{";
-			else {
-				source += `(?:${pattern.slice(index + 1, end).split(",").map((part) => part.replace(/[.+^$()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*")).join("|")})`;
-				index = end;
-			}
-		} else source += char.replace(/[.+^$()|[\]\\]/g, "\\$&");
-	}
-	const expression = new RegExp(`^${source}$`);
-	return (path) => expression.test(path);
-}
-//#endregion
-//#region ../runner/src/snapshot.ts
-const GLOB_CHARS = /[*?{]/;
-function ignoreRule(pattern) {
-	const negated = pattern.startsWith("!");
-	const trimmed = (negated ? pattern.slice(1) : pattern).replace(/^\.\//, "").replace(/\/$/, "");
-	const match = globMatcher(trimmed);
-	if (!trimmed.includes("/")) return {
-		negated,
-		match: (relativePath) => match(path.posix.basename(relativePath)),
-		mayMatchBelow: () => true
-	};
-	const segments = trimmed.split("/");
-	const firstGlob = segments.findIndex((segment) => GLOB_CHARS.test(segment));
-	const prefix = segments.slice(0, firstGlob === -1 ? segments.length : firstGlob).join("/");
-	return {
-		negated,
-		match,
-		mayMatchBelow: (directory) => prefix === "" || prefix.startsWith(`${directory}/`) || prefix === directory || directory.startsWith(`${prefix}/`)
-	};
-}
-/**
-* Ignore rules for candidate snapshots, read like `.gitignore`: globs relative
-* to the workspace root, where a pattern without `/` matches that name at any
-* depth, and later patterns win. `!pattern` re-includes a path; unlike Git, it
-* also works below an ignored directory (`_build` then `!_build/doc/**`).
-*/
-function ignoreRules(patterns) {
-	const rules = patterns.map(ignoreRule);
-	return {
-		/** Whether a path is ignored, given whether its directory was. */
-		ignored(relativePath, parentIgnored = false) {
-			let ignored = parentIgnored;
-			for (const rule of rules) if (rule.match(relativePath)) ignored = !rule.negated;
-			return ignored;
-		},
-		/** Whether an ignored directory must still be walked for re-included paths. */
-		mayReinclude(directory) {
-			return rules.some((rule) => rule.negated && rule.mayMatchBelow(directory));
-		}
-	};
-}
-/**
-* Copies the candidate workspace's regular files into the run report,
-* leaving out ignored paths. Symbolic links and special files are skipped,
-* never followed, so a snapshot cannot read outside the run. An ignored
-* directory is walked only when a `!` pattern could re-include something in
-* it, and is listed only if it ends up containing a snapshotted file.
-*/
-async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAULT_ARTIFACT_EXCLUDE) {
-	const artifacts = [];
-	const rules = ignoreRules(ignore);
-	/** Returns whether anything below `directory` was snapshotted. */
-	async function visit(directory, relativeDirectory, directoryIgnored) {
-		const entries = await readdir(directory, { withFileTypes: true });
-		entries.sort((left, right) => left.name.localeCompare(right.name));
-		let kept = false;
-		for (const entry of entries) {
-			const absolutePath = path.join(directory, entry.name);
-			const relativePath = path.join(relativeDirectory, entry.name);
-			const posixPath = relativePath.split(path.sep).join("/");
-			const ignored = rules.ignored(posixPath, directoryIgnored);
-			const stat = await lstat(absolutePath);
-			if (stat.isSymbolicLink()) continue;
-			if (stat.isDirectory()) {
-				if (ignored && !rules.mayReinclude(posixPath)) continue;
-				const entryIndex = artifacts.length;
-				const keptBelow = await visit(absolutePath, relativePath, ignored);
-				if (!ignored || keptBelow) {
-					artifacts.splice(entryIndex, 0, {
-						path: path.join("candidate", relativePath),
-						kind: "directory"
-					});
-					kept = true;
-				}
-				continue;
-			}
-			if (ignored || !stat.isFile()) continue;
-			const data = await readFile(absolutePath);
-			const artifactPath = path.join("candidate", relativePath);
-			await writer.writeArtifact(artifactPath, data);
-			artifacts.push({
-				path: artifactPath,
-				kind: "file",
-				size: data.byteLength
-			});
-			kept = true;
-		}
-		return kept;
-	}
-	await visit(candidateRoot, "", false);
-	return artifacts;
-}
-//#endregion
-//#region ../runner/src/datasets.ts
-const loads = /* @__PURE__ */ new WeakMap();
-/**
-* Loads a dataset at most once per process and project root. A failed load is
-* forgotten, so the next run that needs the dataset tries again.
-*/
-function loadDataset(dataset, root, signal) {
-	let byRoot = loads.get(dataset);
-	if (!byRoot) loads.set(dataset, byRoot = /* @__PURE__ */ new Map());
-	const cached = byRoot.get(root);
-	if (cached) return cached;
-	const loading = loadUncached(dataset, root, signal);
-	byRoot.set(root, loading);
-	loading.catch(() => {
-		if (byRoot.get(root) === loading) byRoot.delete(root);
-	});
-	return loading;
-}
-async function loadUncached(dataset, root, signal) {
-	const { source } = dataset;
-	if (source.kind === "computed") {
-		const loaded = await source.onLoad({
-			root,
-			signal
-		});
-		if (typeof loaded !== "object" || loaded === null) throw new Error(`Dataset ${dataset.id} onLoad must return { files?, value? }`);
-		return loaded;
-	}
-	return {
-		...source.files !== void 0 ? { files: source.files } : {},
-		...source.valueFile !== void 0 ? { value: await readValueFile(dataset.id, path.resolve(root, source.valueFile)) } : source.value !== void 0 ? { value: source.value } : {}
-	};
-}
-/** `.json` is one value; `.jsonl` is an array of one JSON object per non-blank line. */
-async function readValueFile(id, file) {
-	const text = await readFile(file, "utf8").catch((cause) => {
-		throw new Error(`Dataset ${id} cannot read ${file}: ${String(cause)}`);
-	});
-	if (file.endsWith(".jsonl")) return text.split("\n").flatMap((line, index) => {
-		if (!line.trim()) return [];
-		try {
-			const row = JSON.parse(line);
-			if (typeof row !== "object" || row === null || Array.isArray(row)) throw new Error("expected a JSON object");
-			return [row];
-		} catch (cause) {
-			throw new Error(`Dataset ${id} ${file}:${index + 1}: ${cause instanceof Error ? cause.message : String(cause)}`);
-		}
-	});
-	if (file.endsWith(".json")) try {
-		return JSON.parse(text);
-	} catch (cause) {
-		throw new Error(`Dataset ${id} ${file}: ${cause instanceof Error ? cause.message : String(cause)}`);
-	}
-	throw new Error(`Dataset ${id} valueFile must be a .json or .jsonl file: ${file}`);
-}
-/** Loads (cached), derives this run's content with `onRun`, and resolves it. */
-async function prepareDatasets(datasets, root, run) {
-	const prepared = [];
-	for (const dataset of datasets) {
-		const loaded = await loadDataset(dataset, root, run.signal);
-		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, run) : loaded;
-		if (typeof content !== "object" || content === null) throw new Error(`Dataset ${dataset.id} onRun must return { files?, value? }`);
-		prepared.push(await resolveContent(dataset, content, root));
-	}
-	return prepared;
-}
-async function resolveContent(dataset, content, root) {
-	const label = `Dataset ${dataset.id}`;
-	const value = content.value === void 0 ? void 0 : toJson(content.value, `datasets.${dataset.id}.value`);
-	let files = { kind: "none" };
-	if (typeof content.files === "string") {
-		const source = path.resolve(root, content.files);
-		if (!(await stat(source).catch(() => void 0))?.isDirectory()) throw new Error(`${label} files directory does not exist: ${content.files}`);
-		files = {
-			kind: "directory",
-			source
-		};
-	} else if (content.files !== void 0) files = {
-		kind: "entries",
-		entries: Object.entries(content.files).map(([file, source]) => preparedFile(relativeDatasetPath(file, `${label} file`), source, root, label))
-	};
-	return {
-		dataset,
-		files,
-		...value === void 0 ? {} : { value },
-		digest: await digest(files, value)
-	};
-}
-function preparedFile(file, source, root, label) {
-	if (typeof source === "string") return {
-		path: file,
-		kind: "bytes",
-		data: new TextEncoder().encode(source)
-	};
-	if (source instanceof Uint8Array) return {
-		path: file,
-		kind: "bytes",
-		data: source
-	};
-	if (typeof source === "object" && source !== null && typeof source.file === "string") return {
-		path: file,
-		kind: "file",
-		source: path.resolve(root, source.file)
-	};
-	throw new Error(`${label} file ${file} must be a string, bytes, or { file: 'path' }`);
-}
-/** Lists a directory's files as sorted relative POSIX paths. */
-async function listFiles(directory, prefix = "") {
-	const entries = await readdir(path.join(directory, prefix), { withFileTypes: true });
-	const files = [];
-	for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
-		const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
-		if (entry.isDirectory()) files.push(...await listFiles(directory, relative));
-		else if (entry.isFile()) files.push(relative);
-	}
-	return files;
-}
-async function digest(files, value) {
-	const hash = createHash("sha256");
-	const addFile = (file, data) => {
-		hash.update(`file ${file} ${data.byteLength}\n`);
-		hash.update(data);
-	};
-	if (files.kind === "directory") for (const file of await listFiles(files.source)) addFile(file, await readFile(path.join(files.source, file)));
-	else if (files.kind === "entries") for (const entry of [...files.entries].sort((a, b) => a.path.localeCompare(b.path))) addFile(entry.path, entry.kind === "bytes" ? entry.data : await readFile(entry.source));
-	if (value !== void 0) hash.update(`value ${canonicalParameters(value)}\n`);
-	return `sha256:${hash.digest("hex")}`;
-}
-function runDatasets(prepared) {
-	return prepared.map(({ dataset, digest }) => ({
-		id: dataset.id,
-		visibility: dataset.visibility,
-		source: dataset.source.kind,
-		digest
-	}));
-}
-/**
-* The files a static dataset places in every workspace, relative to its
-* mount. Computed datasets return `undefined`: their files only exist once
-* `onLoad` runs, and browsing must never run project code.
-*/
-async function staticDatasetFiles(dataset, root) {
-	const { source } = dataset;
-	if (source.kind === "computed") return void 0;
-	if (source.files === void 0) return [];
-	if (typeof source.files === "string") {
-		const directory = path.resolve(root, source.files);
-		return Promise.all((await listFiles(directory)).map(async (file) => ({
-			path: file,
-			size: (await stat(path.join(directory, file))).size
-		})));
-	}
-	return Promise.all(Object.entries(source.files).sort(([a], [b]) => a.localeCompare(b)).map(async ([file, content]) => ({
-		path: file,
-		size: typeof content === "string" ? new TextEncoder().encode(content).byteLength : content instanceof Uint8Array ? content.byteLength : (await stat(path.resolve(root, content.file))).size
-	})));
-}
-/** One file of a static dataset, by its path below the mount. */
-async function staticDatasetFile(dataset, root, file) {
-	const { source } = dataset;
-	const missing = () => /* @__PURE__ */ new Error(`Dataset ${dataset.id} has no file ${JSON.stringify(file)}`);
-	if (source.kind === "computed" || source.files === void 0) throw missing();
-	if (typeof source.files === "string") {
-		const directory = path.resolve(root, source.files);
-		const resolved = path.resolve(directory, file);
-		if (!resolved.startsWith(`${directory}${path.sep}`)) throw missing();
-		if (!(await stat(resolved).catch(() => void 0))?.isFile()) throw missing();
-		return readFile(resolved);
-	}
-	if (!Object.hasOwn(source.files, file)) throw missing();
-	const content = source.files[file];
-	if (typeof content === "string") return new TextEncoder().encode(content);
-	if (content instanceof Uint8Array) return new Uint8Array(content);
-	return readFile(path.resolve(root, content.file));
-}
-//#endregion
-//#region ../runner/src/workspace.ts
-/** Where a run's persistent workspace lives below `root`. */
-function runWorkspacePath(root, runId) {
-	return path.join(path.resolve(root), runId.replace(/[^a-zA-Z0-9._-]/g, "_"));
-}
-/**
-* Creates isolated candidate and evaluator directories and writes each
-* dataset's files below its mount, on the side its visibility allows.
-* Two datasets writing the same file is an error, never an overwrite.
-*/
-async function createRunWorkspace(runId, datasets = [], persistentRoot) {
-	const root = persistentRoot ? runWorkspacePath(persistentRoot, runId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-run-"));
-	const persistent = persistentRoot !== void 0;
-	const artifacts = {
-		candidate: { root: path.join(root, "candidate") },
-		evaluator: { root: path.join(root, "evaluator") }
-	};
-	try {
-		await Promise.all([mkdir(artifacts.candidate.root, { recursive: true }), mkdir(artifacts.evaluator.root, { recursive: true })]);
-		for (const prepared of datasets) {
-			const side = prepared.dataset.visibility === "candidate" ? artifacts.candidate.root : artifacts.evaluator.root;
-			await writeDatasetFiles(prepared, path.join(side, prepared.dataset.mount));
-		}
-		return {
-			artifacts,
-			cleanup: async (keep = false) => {
-				if (persistent && keep) return;
-				await rm(root, {
-					recursive: true,
-					force: true
-				});
-			}
-		};
-	} catch (error) {
-		await rm(root, {
-			recursive: true,
-			force: true
-		});
-		throw error;
-	}
-}
-async function writeDatasetFiles(prepared, target) {
-	const { files, dataset } = prepared;
-	if (files.kind === "none") return;
-	await mkdir(target, { recursive: true });
-	const entries = files.kind === "directory" ? (await listFiles(files.source)).map((file) => ({
-		path: file,
-		kind: "file",
-		source: path.join(files.source, file)
-	})) : files.entries;
-	for (const entry of entries) await writeEntry(dataset.id, target, entry);
-}
-async function writeEntry(datasetId, target, entry) {
-	const destination = path.join(target, ...entry.path.split("/"));
-	await mkdir(path.dirname(destination), { recursive: true });
-	await (entry.kind === "bytes" ? writeFile(destination, entry.data, { flag: "wx" }) : copyFile(entry.source, destination, 1)).catch((cause) => {
-		throw collision(datasetId, entry.path, cause);
-	});
-}
-function collision(datasetId, file, cause) {
-	return cause.code === "EEXIST" ? /* @__PURE__ */ new Error(`Dataset ${datasetId} would overwrite ${file}, which another dataset already wrote`) : cause instanceof Error ? cause : new Error(String(cause));
-}
-//#endregion
-//#region ../runner/src/checkpoint-lookup.ts
-/**
-* `checkpoint(rule)` for a scoring context: the latest result of the steps
-* that are this very rule. A rule outside the transcript is a mistake in the
-* scenario, so it throws rather than reading as "not run".
-*/
-function checkpointLookup(transcript, results) {
-	return ((rule) => {
-		const steps = new Set(transcript.flatMap((step, index) => step === rule ? [index] : []));
-		if (!steps.size) throw new Error(`checkpoint(${JSON.stringify(rule.name)}) is not a checkpoint in this scenario's transcript; pass the same rule object the transcript uses`);
-		for (let index = results.length - 1; index >= 0; index--) if (steps.has(results[index].step)) return results[index];
-	});
-}
-//#endregion
-//#region ../runner/src/score.ts
-/**
-* Shared normalization for inline and final predicate and judge rules.
-* `passAt` turns a numeric value into a verdict; an explicit `passed` wins.
-*/
-function normalizeScore(value, passAt) {
-	if (typeof value === "object" && value !== null && "skipped" in value && value.skipped === true) return {
-		skipped: true,
-		explanation: value.reason
-	};
-	const normalized = typeof value === "boolean" ? { value: Number(value) } : typeof value === "number" ? { value } : value;
-	if (!normalized || !Number.isFinite(normalized.value) || normalized.value < 0 || normalized.value > 1) throw new Error(`Score value must be a finite number from 0 to 1; received ${normalized?.value}`);
-	if ("judge" in normalized && normalized.judge) {
-		const { usage } = normalized.judge;
-		if (usage && Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
-	}
-	const passed = "passed" in normalized && normalized.passed !== void 0 ? normalized.passed : normalized.value >= (passAt ?? 1);
-	return {
-		value: normalized.value,
-		passed,
-		..."explanation" in normalized && normalized.explanation ? { explanation: normalized.explanation } : {},
-		..."evidence" in normalized && normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
-		..."judge" in normalized && normalized.judge !== void 0 ? { judge: normalized.judge } : {}
-	};
-}
-//#endregion
-//#region ../runner/src/judge-agent.ts
-const failure$2 = (name, cause) => new RuleExecutionError({
-	name,
-	cause,
-	message: cause instanceof Error ? cause.message : String(cause)
-});
-/** Drive a distinct agent session for one assessment; do not add its events to AUT trajectory. */
-function runJudgeAgent(rule, agent, input, placement) {
-	const events = [];
-	const context = {
-		...input.context,
-		data: input.data,
-		workspace: input.artifacts.evaluator,
-		evaluatorWorkspace: input.artifacts.evaluator
-	};
-	const prompt = JSON.stringify({
-		name: rule.name,
-		rubric: rule.rubric,
-		placement,
-		evidence: placement === "transcript" ? { turn: input.turn?.events ?? [] } : { trajectory: input.trajectory.events },
-		instruction: "Return only a JSON verdict with value (a number from 0 to 1), optional passed, explanation, and evidence."
-	});
-	let closeError;
-	return Effect.gen(function* () {
-		yield* Effect.acquireUseRelease(Effect.tryPromise({
-			try: () => agent.start({
-				context,
-				attach: input.attach,
-				onEvent: (event) => {
-					const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
-						...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
-						source: "aut"
-					});
-					if (observed.source !== "aut") throw new Error("Invalid judge event source");
-					const { source: _, ...validated } = observed;
-					events.push(validated);
-				}
-			}),
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => {
-				const result = await session.send(prompt);
-				if (result !== void 0) events.push(...turnResultEvents(result, 1, () => /* @__PURE__ */ new Date()));
-			},
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => session.close?.(),
-			catch: (cause) => failure$2(rule.name, cause)
-		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
-			closeError = error;
-		}))));
-		if (closeError) return yield* Effect.fail(closeError);
-		const completed = events.slice().reverse().find((event) => event.kind === "completed" && event.output !== void 0);
-		const assistant = events.slice().reverse().find((event) => event.kind === "message" && event.role === "assistant");
-		const output = completed?.kind === "completed" ? completed.output : assistant?.kind === "message" ? assistant.content : void 0;
-		if (output === void 0) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge agent did not emit a verdict")));
-		const value = yield* Effect.try({
-			try: () => typeof output === "string" ? JSON.parse(output) : output,
-			catch: (cause) => failure$2(rule.name, cause)
-		});
-		const encodedEvents = yield* Schema.decodeUnknown(JsonValueSchema)(events).pipe(Effect.mapError((cause) => failure$2(rule.name, cause)));
-		if (!Array.isArray(encodedEvents)) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge events must be an array")));
-		const usage = events.slice().reverse().find((event) => event.kind === "turn-completed" && event.usage);
-		return {
-			value,
-			judge: {
-				...agent.identity ? { agent: agent.identity } : {},
-				...usage?.kind === "turn-completed" && usage.usage ? { usage: usage.usage } : {},
-				events: encodedEvents
-			}
-		};
-	});
-}
-//#endregion
-//#region ../runner/src/evaluate-rule.ts
-const failure$1 = (name, cause) => new RuleExecutionError({
-	name,
-	cause,
-	message: cause instanceof Error ? cause.message : String(cause)
-});
-/** Both placements call the same rule executor; only input timing and persistence differ. */
-function evaluateRule(rule, input, judgeAgent, placement) {
-	return Effect.gen(function* () {
-		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("Judge rules require an eval judge agent")));
-		const evaluated = rule.kind === "predicate" ? { value: yield* Effect.tryPromise({
-			try: async () => rule.run(input),
-			catch: (cause) => failure$1(rule.name, cause)
-		}) } : yield* runJudgeAgent(rule, judgeAgent, input, placement);
-		const score = yield* Effect.try({
-			try: () => normalizeScore(evaluated.value, rule.kind === "predicate" ? rule.passAt : void 0),
-			catch: (cause) => failure$1(rule.name, cause)
-		});
-		const withJudge = rule.kind === "judge" && "judge" in evaluated ? {
-			...score,
-			judge: evaluated.judge
-		} : score;
-		const validated = yield* Schema.decodeUnknown(ScoreResultSchema)({
-			name: rule.name,
-			kind: rule.kind,
-			durationMs: 0,
-			...withJudge
-		}).pipe(Effect.mapError((cause) => failure$1(rule.name, cause)));
-		if (validated.skipped) return {
-			skipped: true,
-			...validated.explanation === void 0 ? {} : { explanation: validated.explanation }
-		};
-		return {
-			value: validated.value,
-			passed: validated.passed,
-			...validated.explanation === void 0 ? {} : { explanation: validated.explanation },
-			...validated.evidence === void 0 ? {} : { evidence: validated.evidence },
-			...validated.judge === void 0 ? {} : { judge: validated.judge }
-		};
-	});
-}
-//#endregion
-//#region ../runner/src/turn.ts
-/** Freeze the observation window at send() completion, before checkpoint events arrive. */
-function turnView(events, start, end, userStepIndex) {
-	const window = events.slice(start, end);
-	const autEvents = window.filter((event) => event.source === "aut");
-	const assistantMessages = autEvents.filter((event) => event.kind === "message" && event.role === "assistant");
-	const calls = window.flatMap((event, index) => event.source === "aut" && event.kind === "tool-call" ? [{
-		event,
-		eventIndex: start + index
-	}] : []);
-	const results = autEvents.filter((event) => event.kind === "tool-result");
-	return {
-		userStepIndex,
-		events: autEvents,
-		assistantMessages,
-		toolCalls: calls.map(({ event, eventIndex }) => {
-			const matches = results.filter((result) => result.id === event.id);
-			const ambiguous = calls.filter((call) => call.event.id === event.id).length !== 1 || matches.length > 1;
-			return {
-				eventIndex,
-				id: event.id,
-				name: event.name,
-				arguments: event.arguments,
-				resultObservation: ambiguous ? "ambiguous" : matches.length ? "observed" : "absent",
-				...!ambiguous && matches.length ? { result: matches[0].result } : {}
-			};
-		}),
-		...typeof assistantMessages.at(-1)?.content === "string" ? { lastAssistantText: assistantMessages.at(-1).content } : {}
-	};
-}
-/** Exact JSON matching: object keys do not matter, array order does. */
-function matchingToolCall(turn, name, args) {
-	const expected = args === void 0 ? void 0 : canonicalParameters(args);
-	return turn.toolCalls.find((call) => call.name === name && (expected === void 0 || canonicalParameters(call.arguments) === expected));
-}
-//#endregion
-//#region ../runner/src/checkpoint.ts
-const failure = (step, name, cause) => new CheckpointExecutionError({
-	step,
-	name,
-	cause,
-	message: cause instanceof Error ? cause.message : String(cause)
-});
-/** An assertion failure is a successful Effect containing a failed result; only execution errors enter the error channel. */
-function evaluateCheckpoint(step, index, input, now, judgeAgent) {
-	const annotations = {
-		evalId: input.context.evalId,
-		runId: input.context.runId,
-		step: index,
-		checkpoint: step.name
-	};
-	return Effect.gen(function* () {
-		const started = now();
-		yield* Effect.logDebug("checkpoint started");
-		const match = step.kind === "expect-tool-call" ? yield* Effect.try({
-			try: () => matchingToolCall(input.turn, step.expected.name, step.expected.arguments),
-			catch: (cause) => failure(index, step.name, cause)
-		}) : void 0;
-		const score = step.kind === "expect-tool-call" ? yield* Effect.try({
-			try: () => normalizeScore(Boolean(match)),
-			catch: (cause) => failure(index, step.name, cause)
-		}) : yield* evaluateRule(step, input, judgeAgent, "transcript").pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
-		const status = score.skipped ? "skipped" : score.passed ? "passed" : "failed";
-		const { skipped: _, ...recorded } = score;
-		const result = yield* Schema.decodeUnknown(CheckpointResultSchema)({
-			step: index,
-			kind: step.kind,
-			name: step.name,
-			status,
-			durationMs: now().getTime() - started.getTime(),
-			...recorded,
-			...match ? { matchedToolCall: {
-				eventIndex: match.eventIndex,
-				id: match.id
-			} } : {}
-		}).pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
-		yield* Effect.logDebug("checkpoint completed").pipe(Effect.annotateLogs({ status: result.status }));
-		return {
-			...result,
-			status
-		};
-	}).pipe(Effect.tapError((error) => Effect.logDebug("checkpoint execution error").pipe(Effect.annotateLogs({ errorName: error._tag }))), Effect.annotateLogs(annotations));
-}
-//#endregion
-//#region ../runner/src/effect-boundary.ts
-/** Keep tagged Effect failures intact when bridging into the legacy async reporting lifecycle. Defects still reject. */
-async function runBoundary(effect) {
-	const outcome = await Effect.runPromise(Effect.either(effect));
-	if (Either.isLeft(outcome)) throw outcome.left;
-	return outcome.right;
-}
-//#endregion
-//#region ../runner/src/transcript.ts
-async function executeTranscript(definition, session, input) {
-	const { now, emitRunner, progress } = input;
-	let turn = 0;
-	for (const [index, step] of definition.transcript.entries()) {
-		if (input.signal.aborted) throw cancelled();
-		const message = step.kind === "user" ? userMessage(step, index, input) : void 0;
-		await emitRunner({
-			kind: "transcript-step-started",
-			step: index,
-			...message === void 0 ? {} : { message },
-			timestamp: now().toISOString()
-		});
-		if (message !== void 0) {
-			const cursor = input.events.length;
-			turn++;
-			const result = await sendUserTurn(session, message, input.signal);
-			if (result !== void 0) for (const event of turnResultEvents(result, turn, now)) await input.emitAut(event);
-			progress.lastTurn = turnView(input.events, cursor, input.events.length, index);
-		} else if (step.kind === "predicate" || step.kind === "judge" || step.kind === "expect-tool-call") {
-			const result = await runCheckpoint(definition, step, index, input);
-			progress.checkpoints.push(result);
-			await emitRunner({
-				kind: "checkpoint-completed",
-				step: index,
-				status: result.status,
-				timestamp: now().toISOString()
-			});
-			await emitRunner({
-				kind: "transcript-step-completed",
-				step: index,
-				timestamp: now().toISOString()
-			});
-			if (result.status === "failed" && definition.policy?.failfast) {
-				progress.stoppedEarly = true;
-				await skipRemainingSteps(definition, index, input);
-				break;
-			}
-			continue;
-		} else throw new Error(`Transcript step kind "${step.kind}" is not executable yet`);
-		await emitRunner({
-			kind: "transcript-step-completed",
-			step: index,
-			timestamp: now().toISOString()
-		});
-	}
-}
-/** A computed message sees the parameters and candidate data the agent also sees. */
-function userMessage(step, index, { context, randomSeed }) {
-	if (typeof step.message === "string") return step.message;
-	const message = step.message({
-		parameters: context.parameters ?? {},
-		data: context.data,
-		seed: randomSeed,
-		...context.batch ? { batch: context.batch } : {}
-	});
-	if (typeof message !== "string" || !message.trim()) throw new Error(`The user message at step ${index + 1} must compute a non-empty string`);
-	return message;
-}
-async function sendUserTurn(session, message, signal) {
-	return runBoundary(Effect.tryPromise({
-		try: () => unlessAborted(() => session.send(message), signal),
-		catch: (cause) => cause instanceof ReportError || cause instanceof CancelledError ? cause : new AutExecutionError({
-			cause,
-			message: cause instanceof Error ? cause.message : String(cause)
-		})
-	}));
-}
-async function runCheckpoint(definition, step, index, input) {
-	const { now, emitRunner, progress } = input;
-	if (!progress.lastTurn) throw new Error("Checkpoint requires a preceding user step");
-	const started = now();
-	await emitRunner({
-		kind: "checkpoint-started",
-		step: index,
-		name: step.name,
-		timestamp: started.toISOString()
-	});
-	return runBoundary(evaluateCheckpoint(step, index, {
-		context: input.context,
-		artifacts: input.artifacts,
-		trajectory: { events: input.events },
-		turn: progress.lastTurn,
-		...turnOutput(progress.lastTurn),
-		data: input.data,
-		checkpoints: [...progress.checkpoints],
-		checkpoint: checkpointLookup(definition.transcript, [...progress.checkpoints]),
-		attach: input.attach
-	}, now, definition.judge)).catch(async (error) => {
-		const recorded = recordError(error);
-		progress.checkpoints.push({
-			step: index,
-			kind: step.kind,
-			name: step.name,
-			status: "error",
-			durationMs: now().getTime() - started.getTime(),
-			error: recorded
-		});
-		try {
-			await emitRunner({
-				kind: "checkpoint-error",
-				step: index,
-				error: recorded,
-				timestamp: now().toISOString()
-			});
-		} catch {}
-		throw error;
-	});
-}
-async function skipRemainingSteps(definition, after, { now, emitRunner, progress }) {
-	for (let index = after + 1; index < definition.transcript.length; index++) {
-		await emitRunner({
-			kind: "transcript-step-skipped",
-			step: index,
-			timestamp: now().toISOString()
-		});
-		const step = definition.transcript[index];
-		if (step.kind === "predicate" || step.kind === "judge" || step.kind === "expect-tool-call") progress.checkpoints.push({
-			step: index,
-			kind: step.kind,
-			name: step.name,
-			status: "skipped"
-		});
-	}
-}
-/** The scorer-facing `output`: the content of the turn's last assistant message. */
-function turnOutput(turn) {
-	const last = turn?.assistantMessages.at(-1);
-	return last === void 0 ? {} : { output: last.content };
-}
-//#endregion
-//#region ../runner/src/final-scoring.ts
-function scoreSummary(results, checkpoints, skippedScorers, metrics = []) {
-	const valid = results.filter((result) => result.value !== void 0 && !result.skipped);
-	const hasError = results.some((result) => result.error !== void 0);
-	return {
-		results,
-		...metrics.length ? { metrics } : {},
-		...checkpoints.length ? { checkpoints } : {},
-		...skippedScorers.length ? { skippedScorers } : {},
-		...valid.length === 0 ? {} : { overall: valid.reduce((sum, result) => sum + (result.value ?? 0), 0) / valid.length },
-		passed: !hasError && results.every((result) => result.skipped || result.passed === true) && checkpoints.every((result) => result.status === "passed" || result.status === "skipped") && !skippedScorers.length
-	};
-}
-/** What final scorers and metrics receive. */
-function scoringContext(input) {
-	return {
-		context: input.context,
-		artifacts: input.workspace.artifacts,
-		trajectory: { events: input.events },
-		...input.lastTurn ? { turn: input.lastTurn } : {},
-		...turnOutput(input.lastTurn),
-		data: input.data,
-		checkpoints: input.checkpoints,
-		checkpoint: checkpointLookup(input.transcript, input.checkpoints),
-		attach: input.attach
-	};
-}
-/** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
-async function evaluateFinalScorers(definition, input) {
-	const { context, workspace, results, skippedScorers, now, emitRunner } = input;
-	for (const scorer of definition.scoring) {
-		if (input.cancelled || input.reportingFailed || !context || !workspace || (input.primaryError || input.stoppedEarly) && (!("supportsPartial" in scorer) || !scorer.supportsPartial)) {
-			skippedScorers.push(scorer.name);
-			continue;
-		}
-		const started = now();
-		await emitRunner({
-			kind: "scorer-started",
-			scorer: scorer.name,
-			timestamp: started.toISOString()
-		});
-		let result;
-		try {
-			const score = await runBoundary(evaluateRule(scorer, scoringContext({
-				...input,
-				context,
-				workspace,
-				transcript: definition.transcript
-			}), definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
-				cause,
-				message: cause.message
-			}))));
-			result = {
-				name: scorer.name,
-				kind: scorer.kind,
-				durationMs: now().getTime() - started.getTime(),
-				...score
-			};
-			results.push(result);
-		} catch (error) {
-			const recorded = recordError(error);
-			results.push({
-				name: scorer.name,
-				kind: scorer.kind,
-				durationMs: now().getTime() - started.getTime(),
-				error: recorded,
-				passed: false
-			});
-			await emitRunner({
-				kind: "scorer-failed",
-				scorer: scorer.name,
-				error: recorded,
-				timestamp: now().toISOString()
-			});
-			continue;
-		}
-		await emitRunner({
-			kind: "scorer-completed",
-			scorer: scorer.name,
-			...result.skipped ? { skipped: true } : { value: result.value ?? 0 },
-			timestamp: now().toISOString()
-		});
-	}
-}
-//#endregion
-//#region ../runner/src/metrics.ts
-/**
-* What EvalKit measures for every run from its trajectory: the agent's time
-* (first event to the last one before final scoring), tokens when the agent
-* reported usage, and tool calls.
-*/
-function builtInMetrics(events) {
-	const times = events.filter((event) => !event.kind.startsWith("scorer-")).map((event) => Date.parse(event.timestamp)).filter((time) => !Number.isNaN(time));
-	const metrics = [];
-	if (times.length) metrics.push({
-		name: "agent time",
-		value: Math.max(...times) - Math.min(...times),
-		unit: "ms",
-		better: "lower",
-		builtIn: true
-	});
-	let tokens;
-	for (const event of events) if (event.source === "aut" && event.kind === "turn-completed" && event.usage) {
-		const turn = turnTokens(event.usage);
-		if (turn !== void 0) tokens = (tokens ?? 0) + turn;
-	}
-	if (tokens !== void 0) metrics.push({
-		name: "tokens",
-		value: tokens,
-		unit: "tokens",
-		better: "lower",
-		builtIn: true
-	});
-	metrics.push({
-		name: "tool calls",
-		value: events.filter((event) => event.source === "aut" && event.kind === "tool-call").length,
-		builtIn: true
-	});
-	return metrics;
-}
-/** Runs the scenario's metrics; a failing metric records its error and the rest still run. */
-async function evaluateMetrics(definition, context) {
-	const results = [];
-	for (const entry of definition.metrics ?? []) {
-		const shape = {
-			name: entry.name,
-			...entry.unit ? { unit: entry.unit } : {},
-			...entry.better ? { better: entry.better } : {}
-		};
-		try {
-			const value = await entry.run(context);
-			if (value !== void 0 && !Number.isFinite(value)) throw new Error(`Metric ${entry.name} must return a finite number or undefined; got ${String(value)}`);
-			results.push({
-				...shape,
-				...value === void 0 ? {} : { value }
-			});
-		} catch (error) {
-			results.push({
-				...shape,
-				error: recordError(error)
-			});
-		}
-	}
-	return results;
-}
-//#endregion
-//#region ../runner/src/local-report-store.ts
-const MANIFEST_FILE = "manifest.json";
-/** The project's evals and suites when the run started. */
-const CATALOG_FILE = "catalog.json";
-async function writeJson(filePath, value) {
-	await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
-}
-async function replaceJson(filePath, value) {
-	const temporary = `${filePath}.${randomUUID()}.tmp`;
-	await writeJson(temporary, value);
-	await rename(temporary, filePath);
-}
-function assertPathSegment(value, name) {
-	if (!value || value === "." || value === ".." || value.includes("/") || value.includes("\\")) throw new Error(`${name} must be a single relative path segment`);
-}
-function artifactPath(root, value) {
-	if (!value || path.isAbsolute(value)) throw new Error(`Artifact path must be a non-empty relative path: ${value}`);
-	const normalized = path.normalize(value);
-	if (normalized === ".." || normalized.startsWith(`..${path.sep}`)) throw new Error(`Artifact path escapes its root: ${value}`);
-	return path.join(root, normalized);
-}
-var LocalRunWriter = class {
-	location;
-	directory;
-	metadata;
-	constructor(location, directory, metadata) {
-		this.location = location;
-		this.directory = directory;
-		this.metadata = metadata;
-	}
-	appendEvent(event) {
-		return appendFile(path.join(this.directory, "trajectory.jsonl"), `${JSON.stringify(Schema.encodeSync(TrajectoryEventSchema)(event))}\n`);
-	}
-	writeScores(scoring) {
-		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(RunScoringSchema)(scoring));
-	}
-	async writeArtifact(relativePath, data) {
-		const target = artifactPath(path.join(this.directory, "artifacts"), relativePath);
-		await mkdir(path.dirname(target), { recursive: true });
-		await writeFile(target, data);
-	}
-	async finalize(summary) {
-		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(RunSummarySchema)(summary));
-		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
-			...Schema.encodeSync(RunMetadataSchema)(this.metadata),
-			status: summary.status
-		});
-	}
-};
-/** Persists each run as a local tree rooted below `rootDirectory`. */
-function localReportStore(rootDirectory) {
-	return { async startRun(metadata, context) {
-		assertPathSegment(metadata.runId, "runId");
-		const directory = path.resolve(rootDirectory, metadata.runId);
-		await mkdir(directory, { recursive: true });
-		if (context?.catalog) await writeJson(path.join(directory, CATALOG_FILE), context.catalog);
-		await writeJson(path.join(directory, MANIFEST_FILE), {
-			...Schema.encodeSync(RunMetadataSchema)(metadata),
-			status: "running"
-		});
-		return new LocalRunWriter(path.relative(process.cwd(), directory), directory, metadata);
-	} };
-}
-//#endregion
-//#region ../runner/src/local-report-reader.ts
-/** The local report boundary for CLI, dashboard, and CI consumers (schema version 6). */
-async function decode(file, schema) {
-	return Schema.decodeUnknownSync(schema)(JSON.parse(await readFile(file, "utf8")));
-}
-const readRunManifest = (root, runId) => decode(join(root, runId, "manifest.json"), RunManifestSchema);
-const readRunSummary = (root, runId) => decode(join(root, runId, "summary.json"), RunSummarySchema);
-const readRunScoring = (root, runId) => decode(join(root, runId, "scoring.json"), RunScoringSchema);
-async function readRunEvents(root, runId) {
-	return parseTrajectoryJsonl(await readFile(join(root, runId, "trajectory.jsonl"), "utf8"));
-}
-/**
-* True when a run's owning process is known to be gone: it ran on this host
-* and its PID no longer exists. Runs from other hosts are never judged.
-*/
-function ownerIsGone(owner) {
-	if (!owner || owner.hostname !== os.hostname()) return false;
-	if (owner.pid === process.pid) return false;
-	try {
-		process.kill(owner.pid, 0);
-		return false;
-	} catch (error) {
-		return error.code === "ESRCH";
-	}
-}
-//#endregion
-//#region ../runner/src/matrix.ts
-/** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
-function runMatrix(matrix, options) {
-	return Effect.gen(function* () {
-		const concurrency = options.concurrency ?? 4;
-		if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Concurrency must be a positive integer");
-		const count = matrix.count(options.selection);
-		const iterator = matrix.cells(options.selection);
-		const summary = {
-			cells: 0,
-			runs: 0,
-			passed: 0,
-			failed: 0,
-			cancelled: 0
-		};
-		yield* Effect.all(Array.from({ length: Math.min(count, concurrency) }, () => Effect.gen(function* () {
-			while (!options.signal?.aborted) {
-				const next = iterator.next();
-				if (next.done) return;
-				const cell = next.value;
-				const runtime = options.runtime ?? [
-					"local",
-					"remote",
-					"sandbox"
-				].find((name) => agentRuntimes(cell.eval.agent)[name]);
-				const suiteId = options.suiteId ?? options.suiteFor?.(cell);
-				const runs = yield* runEval(cell.eval, {
-					...options,
-					...suiteId ? { suiteId } : {},
-					concurrency: 1,
-					runtime,
-					parameters: cell.parameters,
-					matrix: {
-						id: matrix.id,
-						cellKey: cell.key
-					}
-				});
-				summary.cells++;
-				for (const run of runs) {
-					summary.runs++;
-					if (run.status === "cancelled") summary.cancelled++;
-					else if (run.status === "passed") summary.passed++;
-					else summary.failed++;
-				}
-				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, runs));
-			}
-		})), { concurrency });
-		return summary;
-	});
-}
-//#endregion
-//#region ../runner/src/upgrade.ts
-/** A new run being written. */
-const TEMPORARY = ".upgrade-";
-/** An old run set aside while its replacement, with the same ID, moves in. */
-const RETIRED = ".upgraded-";
-const PER_RUN_FILES = ["archived.json"];
-/**
-* Rewrites results from report schema versions 2 to 5 into version 6, in
-* place (RFD0008). An old run of one trial (most of them) becomes a run with
-* the same ID, so links to it keep working. An old run of several trials
-* becomes runs with the trials' IDs, in a batch with the old run's ID.
-*
-* Each new run is written under a temporary name and renamed into place;
-* artifacts are moved, not copied, as the last step before that rename. An
-* old run is removed only after its new runs exist, and one replaced under
-* the same ID is set aside first, so a crash at any point leaves the old run
-* or complete new ones, and re-running finishes the job. Version 6 runs are
-* left alone.
-*/
-async function upgradeReports(root, options = {}) {
-	const report = {
-		upgraded: 0,
-		runs: 0,
-		batches: 0,
-		running: []
-	};
-	let entries;
-	try {
-		entries = await readdir(root);
-	} catch {
-		return report;
-	}
-	if (!options.dryRun) await finishReplacements(root, entries);
-	for (const entry of entries) {
-		if (entry.startsWith(".")) continue;
-		const directory = join(root, entry);
-		const manifest = await readJson(join(directory, "manifest.json"));
-		if (!manifest || !UPGRADABLE_SCHEMA_VERSIONS.includes(manifest.schemaVersion)) continue;
-		const summary = await readJson(join(directory, "summary.json"));
-		if (!summary && !ownerIsGone(manifest.process)) {
-			report.running.push(entry);
-			continue;
-		}
-		const trials = await trialIds(directory);
-		report.upgraded += 1;
-		report.runs += Math.max(trials.length, 1);
-		if (trials.length > 1) report.batches += 1;
-		if (options.dryRun) continue;
-		await upgradeRun(root, entry, manifest, summary, trials);
-	}
-	if (!options.dryRun) await removeLeftovers(root);
-	return report;
-}
-/**
-* Finishes same-ID replacements a crash interrupted: the old run was set
-* aside, and its replacement may or may not have moved in.
-*/
-async function finishReplacements(root, entries) {
-	for (const entry of entries) {
-		if (!entry.startsWith(RETIRED)) continue;
-		const id = entry.slice(10);
-		const retired = join(root, entry);
-		const directory = join(root, id);
-		const replacement = join(root, `${TEMPORARY}${id}`);
-		if (!await exists(directory)) await rename(await exists(replacement) ? replacement : retired, directory);
-		await rm(retired, {
-			recursive: true,
-			force: true
-		});
-	}
-}
-/**
-* Temporary folders nothing will finish. One holding artifacts is kept: it
-* could be the only copy, and an old run would have moved them back.
-*/
-async function removeLeftovers(root) {
-	for (const entry of await readdir(root)) if (entry.startsWith(TEMPORARY) && !await exists(join(root, entry, "artifacts"))) await rm(join(root, entry), {
-		recursive: true,
-		force: true
-	});
-}
-/** Swaps a fully written replacement in for the old run of the same ID. */
-async function replaceInPlace(root, id, replacement) {
-	const retired = join(root, `${RETIRED}${id}`);
-	await rename(join(root, id), retired);
-	await rename(replacement, join(root, id));
-	await rm(retired, {
-		recursive: true,
-		force: true
-	});
-}
-async function upgradeRun(root, oldRunId, runManifest, runSummary, trials) {
-	const directory = join(root, oldRunId);
-	const catalog = await readJson(join(directory, "catalog.json"));
-	const extras = await presentFiles(directory, PER_RUN_FILES);
-	if (!trials.length) {
-		const target = join(root, `${TEMPORARY}${oldRunId}`);
-		await mkdir(target, { recursive: true });
-		const status = runSummary ? "cancelled" : "running";
-		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
-			run: runManifest,
-			runId: oldRunId,
-			status
-		}));
-		if (runSummary) await writeJsonFile(join(target, "summary.json"), {
-			status,
-			endedAt: runSummary.endedAt,
-			...runSummary.durationMs === void 0 ? {} : { durationMs: runSummary.durationMs },
-			...runSummary.error ? { error: runSummary.error } : {}
-		});
-		await copyShared(directory, target, catalog, extras);
-		await replaceInPlace(root, oldRunId, target);
-		return;
-	}
-	const inPlace = trials.length === 1;
-	for (const trialId of trials) {
-		const runId = inPlace ? oldRunId : trialId;
-		const destination = join(root, runId);
-		if (!inPlace && await exists(join(destination, "manifest.json"))) continue;
-		const source = join(directory, "trials", trialId);
-		const trialManifest = await readJson(join(source, "manifest.json")) ?? {};
-		const trialSummary = await readJson(join(source, "summary.json"));
-		const passed = (trialSummary?.scoring)?.passed;
-		const status = upgradeStatus(trialSummary?.status ?? trialManifest.status, passed);
-		const index = typeof trialManifest.trialIndex === "number" ? trialManifest.trialIndex : trials.indexOf(trialId);
-		const target = join(root, `${TEMPORARY}${runId}`);
-		if (!await exists(join(source, "artifacts")) && await exists(join(target, "artifacts"))) await rename(join(target, "artifacts"), join(source, "artifacts"));
-		await rm(target, {
-			recursive: true,
-			force: true
-		});
-		await mkdir(target, { recursive: true });
-		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
-			run: runManifest,
-			trial: trialManifest,
-			runId,
-			...!inPlace ? { batch: {
-				id: oldRunId,
-				index,
-				size: trials.length
-			} } : {},
-			status
-		}));
-		if (trialSummary) await writeJsonFile(join(target, "summary.json"), {
-			...trialSummary,
-			status
-		});
-		const scoring = await readText(join(source, "scoring.json"));
-		if (scoring !== void 0) await writeFile(join(target, "scoring.json"), scoring);
-		const trajectory = await readText(join(source, "trajectory.jsonl"));
-		if (trajectory !== void 0) await writeFile(join(target, "trajectory.jsonl"), upgradeTrajectory(trajectory));
-		await copyShared(directory, target, catalog, extras);
-		if (await exists(join(source, "artifacts"))) await rename(join(source, "artifacts"), join(target, "artifacts"));
-		if (inPlace) await replaceInPlace(root, oldRunId, target);
-		else await rename(target, destination);
-	}
-	if (!inPlace) await rm(directory, {
-		recursive: true,
-		force: true
-	});
-}
-async function copyShared(from, to, catalog, extras) {
-	if (catalog) await writeJsonFile(join(to, "catalog.json"), upgradeCatalog(catalog));
-	for (const file of extras) await cp(join(from, file), join(to, file));
-}
-async function trialIds(directory) {
-	try {
-		return (await readdir(join(directory, "trials"), { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
-	} catch {
-		return [];
-	}
-}
-async function presentFiles(directory, names) {
-	const present = [];
-	for (const name of names) if (await exists(join(directory, name))) present.push(name);
-	return present;
-}
-async function exists(path) {
-	try {
-		await stat(path);
-		return true;
-	} catch {
-		return false;
-	}
-}
-async function readText(path) {
-	try {
-		return await readFile(path, "utf8");
-	} catch {
-		return;
-	}
-}
-async function readJson(path) {
-	const text = await readText(path);
-	if (text === void 0) return void 0;
-	let value;
-	try {
-		value = JSON.parse(text);
-	} catch (error) {
-		throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
-	}
-	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
-}
-async function writeJsonFile(path, value) {
-	await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
-}
-//#endregion
-//#region ../runner/src/index.ts
-/**
-* Runs a scenario: once, or `repeat` times as one batch. Resolves to one
-* result per run started; runs never started because of cancellation are
-* left out.
-*/
-function runEval(definition, options) {
-	return Effect.gen(function* () {
-		yield* Effect.logInfo("eval run started").pipe(Effect.annotateLogs({
-			evalId: authoringId(definition),
-			...options.runId ? { runId: options.runId } : {},
-			...options.suiteId ? { suiteId: options.suiteId } : {}
-		}));
-		const runs = yield* Effect.tryPromise({
-			try: () => executeBatch(definition, options),
-			catch: (error) => error
-		});
-		yield* Effect.logInfo("eval run finished").pipe(Effect.annotateLogs({
-			evalId: authoringId(definition),
-			runs: runs.length,
-			passed: runs.filter((run) => run.status === "passed").length
-		}));
-		return runs;
-	});
-}
-function createId() {
-	return crypto.randomUUID();
-}
-function canonicalId(kind, id) {
-	return resourceUri(kind, id);
-}
-function assertUuid(value, label) {
-	if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error(`${label} must be a UUID`);
-}
-function runMetadata(definition, options, runId, startedAt, extra) {
-	return {
-		schemaVersion: 6,
-		runId,
-		runUri: canonicalId("run", runId),
-		evalId: definition.id,
-		...options.suiteId ? { suiteId: options.suiteId } : {},
-		...options.source ? { source: options.source } : {},
-		process: {
-			pid: process.pid,
-			hostname: os.hostname()
-		},
-		...definition.agent.identity ? { aut: definition.agent.identity } : {},
-		parameters: options.parameters,
-		matrix: options.matrix,
-		startedAt,
-		...options.batch ? { batch: options.batch } : {},
-		...extra
-	};
-}
-/** Recording a secondary error must never mask the original failure or block cleanup. */
-async function tryEmitError(error, emitRunner, now) {
-	try {
-		await emitRunner({
-			kind: "error",
-			error: recordError(error),
-			timestamp: now().toISOString()
-		});
-	} catch {}
-}
-/** Starts `repeat` runs of a scenario, as one batch when there are several. */
-async function executeBatch(definition, options) {
-	let hasUser = false;
-	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
-	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
-	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
-	assertNoRemovedOptions(definition);
-	if (options.runId) assertUuid(options.runId, "runId");
-	const repeat = options.repeat ?? definition.repeat ?? 1;
-	if (!Number.isInteger(repeat) || repeat < 1) throw new Error(`repeat must be a positive integer; received ${repeat}`);
-	if (repeat === 1) return [await runOnce(definition, options)];
-	if (options.runId) throw new Error("runId names one run; it cannot be used with repeat");
-	const batchId = createId();
-	const signal = options.signal;
-	const runs = Array.from({ length: repeat }, (_, index) => {
-		const run = Effect.tryPromise({
-			try: async () => signal?.aborted ? void 0 : runOnce(definition, {
-				...options,
-				batch: {
-					id: batchId,
-					index,
-					size: repeat
-				}
-			}),
-			catch: (error) => error
-		});
-		return options.semaphore ? options.semaphore.withPermits(1)(run) : run;
-	});
-	return (await Effect.runPromise(Effect.all(runs, { concurrency: options.concurrency ?? 32 }))).filter((run) => run !== void 0);
-}
-/** One execution: a session in its own sandbox, scored and reported. */
-async function runOnce(definition, options) {
-	const now = options.now ?? (() => /* @__PURE__ */ new Date());
-	const runId = options.runId ?? createId();
-	const startedAt = now().toISOString();
-	const selectedRuntime = options.runtime;
-	const runtime = selectedRuntime ? agentRuntimes(definition.agent)[selectedRuntime] : void 0;
-	if (selectedRuntime && !runtime) throw new Error(`AUT does not declare the requested ${selectedRuntime} runtime`);
-	const randomSeed = crypto.getRandomValues(/* @__PURE__ */ new Uint32Array(1))[0];
-	const batch = options.batch;
-	const signal = options.signal ?? new AbortController().signal;
-	const root = options.root ?? process.cwd();
-	const baseContext = {
-		runId,
-		evalId: authoringId(definition),
-		...batch ? { batch } : {},
-		metadata: {
-			...toJsonObject(definition.metadata ?? {}, `${definition.id}.metadata`),
-			randomSeed
-		},
-		...options.parameters ? { parameters: options.parameters } : {},
-		...selectedRuntime ? { runtime: selectedRuntime } : {}
-	};
-	let datasetsError;
-	let prepared = [];
-	try {
-		prepared = await prepareDatasets(definition.datasets ?? [], root, {
-			evalId: baseContext.evalId,
-			runId,
-			...batch ? { batch } : {},
-			seed: randomSeed,
-			parameters: options.parameters ?? {},
-			signal
-		});
-	} catch (error) {
-		datasetsError = error;
-	}
-	const candidateData = datasetValues(prepared, "candidate");
-	const scoringData = datasetValues(prepared, "all");
-	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
-	const events = [];
-	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, startedAt, {
-		workspaceIgnore,
-		...prepared.length ? { datasets: runDatasets(prepared) } : {}
-	}), { catalog: options.catalog });
-	const attachments = attachmentStore(runWriter, options.attachmentLimits);
-	let writeChain = Promise.resolve();
-	let reportingFailed = false;
-	const runRef = {
-		runId,
-		...batch ? { batch } : {},
-		evalId: definition.id,
-		...options.suiteId ? { suiteId: options.suiteId } : {},
-		...options.parameters ? { parameters: options.parameters } : {}
-	};
-	const observe = (event) => {
-		try {
-			options.onEvent?.(runRef, event);
-		} catch {}
-	};
-	const append = async (event) => {
-		events.push(event);
-		writeChain = writeChain.then(() => runWriter.appendEvent(event));
-		try {
-			await writeChain;
-			observe(event);
-		} catch (cause) {
-			reportingFailed = true;
-			throw new ReportError({
-				cause,
-				message: cause instanceof Error ? cause.message : String(cause)
-			});
-		}
-	};
-	const emitRunner = (event) => append({
-		...event,
-		source: "runner"
-	});
-	const emitAut = (event) => append({
-		...event,
-		source: "aut"
-	});
-	const onAutEvent = (event) => emitAut(normalizeAutEvent(event, now));
-	let workspace;
-	let context;
-	let primaryError;
-	let closeError;
-	let scoring;
-	let artifacts;
-	let status = "running";
-	const progress = {
-		checkpoints: [],
-		stoppedEarly: false
-	};
-	let cleanupError;
-	const workspaceScope = await Effect.runPromise(Scope.make());
-	try {
-		await emitRunner({
-			kind: "run-started",
-			timestamp: now().toISOString()
-		});
-		if (signal.aborted) throw cancelled();
-		if (datasetsError) throw new DatasetError({
-			cause: datasetsError,
-			message: datasetsError instanceof Error ? datasetsError.message : String(datasetsError)
-		});
-		workspace = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
-			try: () => createRunWorkspace(runId, prepared, options.workspaceRoot),
-			catch: (cause) => new DatasetError({
-				cause,
-				message: cause instanceof Error ? cause.message : String(cause)
-			})
-		}), (created) => Effect.tryPromise({
-			try: () => created.cleanup(keepWorkspace(options.keepWorkspaces, scoring?.passed === true)),
-			catch: (cause) => new DatasetError({
-				cause,
-				message: cause instanceof Error ? cause.message : String(cause)
-			})
-		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
-			cleanupError = error;
-		}).pipe(Effect.tap(() => Effect.logError("run workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
-		const runWorkspace = workspace;
-		const runContext = {
-			...baseContext,
-			data: candidateData,
-			workspace: runWorkspace.artifacts.candidate,
-			evaluatorWorkspace: runWorkspace.artifacts.evaluator
-		};
-		context = runContext;
-		await runBoundary(Effect.acquireUseRelease(Effect.tryPromise({
-			try: () => definition.agent.start({
-				context: runContext,
-				...runtime ? { runtime } : {},
-				onEvent: onAutEvent,
-				attach: attachments.attach
-			}),
-			catch: (cause) => new AutExecutionError({
-				cause,
-				message: cause instanceof Error ? cause.message : String(cause)
-			})
-		}), (session) => Effect.tryPromise({
-			try: () => executeTranscript(definition, session, {
-				context: runContext,
-				artifacts: runWorkspace.artifacts,
-				events,
-				now,
-				randomSeed,
-				emitRunner,
-				emitAut,
-				data: scoringData,
-				signal,
-				progress,
-				attach: attachments.attach
-			}),
-			catch: (cause) => cause
-		}), (opened) => Effect.tryPromise({
-			try: async () => opened.close?.(),
-			catch: (cause) => new AutExecutionError({
-				cause,
-				message: cause instanceof Error ? cause.message : String(cause)
-			})
-		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
-			closeError = error;
-		}).pipe(Effect.tap(() => Effect.logError("AUT session close failed")))))).pipe(Effect.annotateLogs({
-			runId,
-			evalId: definition.id
-		})));
-	} catch (error) {
-		primaryError = error;
-		await tryEmitError(error, emitRunner, now);
-	}
-	if (closeError) {
-		if (!primaryError) primaryError = closeError;
-		await tryEmitError(closeError, emitRunner, now);
-	}
-	const wasCancelled = primaryError instanceof CancelledError;
-	const { checkpoints, stoppedEarly, lastTurn } = progress;
-	const scoreResults = [];
-	const skippedScorers = [];
-	const metrics = [];
-	try {
-		await evaluateFinalScorers(definition, {
-			context,
-			workspace,
-			events,
-			lastTurn,
-			primaryError,
-			stoppedEarly,
-			cancelled: wasCancelled,
-			reportingFailed,
-			results: scoreResults,
-			skippedScorers,
-			data: scoringData,
-			checkpoints,
-			attach: attachments.attach,
-			now,
-			emitRunner
-		});
-		metrics.push(...builtInMetrics(events));
-		if (context && workspace && !wasCancelled) metrics.push(...await evaluateMetrics(definition, scoringContext({
-			context,
-			workspace,
-			events,
-			lastTurn,
-			data: scoringData,
-			checkpoints,
-			transcript: definition.transcript,
-			attach: attachments.attach
-		})));
-		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
-		await runWriter.writeScores(scoring);
-		if (workspace) try {
-			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, runWriter, workspaceIgnore);
-		} catch (captureError) {
-			if (!primaryError) primaryError = captureError;
-			await tryEmitError(captureError, emitRunner, now);
-		}
-	} catch (error) {
-		if (!primaryError) primaryError = error;
-		await tryEmitError(error, emitRunner, now);
-	} finally {
-		await Effect.runPromise(Scope.close(workspaceScope, Exit.succeed(void 0)));
-		if (cleanupError) {
-			if (!primaryError) primaryError = cleanupError;
-			await tryEmitError(cleanupError, emitRunner, now);
-		}
-	}
-	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
-	if (!primaryError) try {
-		await emitRunner({
-			kind: "run-completed",
-			timestamp: now().toISOString()
-		});
-	} catch (error) {
-		primaryError = error;
-	}
-	status = wasCancelled ? "cancelled" : primaryError ? "errored" : scoring.passed ? "passed" : "failed";
-	const error = primaryError ? recordError(primaryError) : void 0;
-	const attached = await attachments.close();
-	const endedAt = now().toISOString();
-	const durationMs = new Date(endedAt).getTime() - new Date(startedAt).getTime();
-	await runWriter.finalize({
-		status,
-		endedAt,
-		durationMs,
-		scoring,
-		...artifacts ? { artifacts } : {},
-		...attached.length ? { attachments: attached } : {},
-		...error ? { error } : {}
-	});
-	return {
-		runId,
-		runUri: canonicalId("run", runId),
-		evalId: definition.id,
-		status,
-		...batch ? { batch } : {},
-		reportLocation: runWriter.location,
-		durationMs,
-		scoring,
-		...error ? { error } : {}
-	};
-}
-function datasetValues(prepared, audience) {
-	const values = {};
-	for (const { dataset, value } of prepared) {
-		if (value === void 0) continue;
-		if (audience === "candidate" && dataset.visibility !== "candidate") continue;
-		values[dataset.id] = value;
-	}
-	return values;
-}
-/** Persistent workspaces of runs that did not pass are kept for debugging by default. */
-function keepWorkspace(policy = "failed", passed) {
-	return policy === "all" || policy === "failed" && !passed;
-}
-//#endregion
-export { DatasetError as _, readRunEvents as a, readRunSummary as c, staticDatasetFile as d, staticDatasetFiles as f, CheckpointExecutionError as g, CancelledError as h, ownerIsGone as i, localReportStore as l, AutExecutionError as m, upgradeReports as n, readRunManifest as o, globMatcher as p, runMatrix as r, readRunScoring as s, runEval as t, runWorkspacePath as u, ReportError as v, ScoringError as y };