@leostera/evalkit 0.8.0 → 0.9.0

package/dist/{src-mm8L_IxM.mjs → src-CcdLZqFk.mjs}

@@ -1,8 +1,9 @@
-import { A as RunSummarySchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, K as toJson, M as TrajectoryEventSchema, N as TrialManifestSchema, P as TrialMetadataSchema, R as normalizeAutEvent, S as JsonValueSchema, V as canonicalParameters, X as relativeDatasetPath, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as ScoreResultSchema, p as recordError, q as toJsonObject, rt as resourceUri, x as CheckpointResultSchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
+import { $ as relativeDatasetPath, A as RunManifestSchema, B as turnResultEvents, C as CheckpointResultSchema, F as RunSummarySchema, G as DEFAULT_ATTACHMENT_LIMITS, H as canonicalParameters, I as ScoreResultSchema, L as TrajectoryEventSchema, M as RunScoringSchema, U as turnTokens, W as DEFAULT_ARTIFACT_EXCLUDE, X as toJsonObject, Y as toJson, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as RunMetadataSchema, ot as resourceUri, p as recordError, w as JsonValueSchema, z as normalizeAutEvent } from "./src-BrfIXfO2.mjs";
+import { a as upgradeManifest, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS } from "./upgrade-60SIRius.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
 import os from "node:os";
-import { appendFile, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { appendFile, copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import path, { join } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 //#region ../runner/src/errors.ts
@@ -12,7 +13,7 @@ var ReportError = class extends Data.TaggedError("ReportError") {};
 var ScoringError = class extends Data.TaggedError("ScoringError") {};
 var RuleExecutionError = class extends Data.TaggedError("RuleExecutionError") {};
 var CheckpointExecutionError = class extends Data.TaggedError("CheckpointExecutionError") {};
-/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
 var CancelledError = class extends Data.TaggedError("CancelledError") {};
 function cancelled() {
 	return new CancelledError({ message: "Cancelled" });
@@ -32,6 +33,79 @@ async function unlessAborted(work, signal) {
 	}
 }
 //#endregion
+//#region ../runner/src/attachments.ts
+const SEGMENT = /^[A-Za-z0-9_-][A-Za-z0-9._-]{0,99}$/;
+const MAX_NAME = 200;
+/**
+* Checks an attachment name: a relative path whose segments are letters,
+* digits, `.`, `_` and `-`, not starting with a dot. That rules out
+* absolute paths, `..`, hidden files, separators other than `/`, and
+* anything a filesystem or URL would read differently.
+*/
+function attachmentName(name) {
+	if (typeof name !== "string" || !name || name.length > MAX_NAME || !name.split("/").every((segment) => SEGMENT.test(segment))) throw new Error(`Attachment name ${JSON.stringify(name)} is not allowed: use a relative path of letters, digits, ".", "_" and "-", like "logs/agent.log"`);
+	return name;
+}
+/** The last `max` bytes, starting on a whole UTF-8 character when the content is text. */
+function lastBytes(bytes, max, text) {
+	let start = bytes.byteLength - max;
+	if (text) while (start < bytes.byteLength && (bytes[start] & 192) === 128) start++;
+	return bytes.subarray(start);
+}
+/**
+* Saves a run's attachments as `attachments/<name>` below its artifacts.
+* Content is stored as given and never interpreted. A file over
+* `maxFileBytes` keeps its end (where a log's errors are); one that would
+* take the run past `maxRunBytes` is rejected.
+*/
+function attachmentStore(writer, limits = {}) {
+	const { maxFileBytes, maxRunBytes } = {
+		...DEFAULT_ATTACHMENT_LIMITS,
+		...limits
+	};
+	const entries = [];
+	const names = /* @__PURE__ */ new Set();
+	const pending = /* @__PURE__ */ new Set();
+	let used = 0;
+	let closed = false;
+	async function attach(name, content) {
+		if (closed) throw new Error(`Cannot attach ${JSON.stringify(name)}: the run has finished`);
+		attachmentName(name);
+		if (typeof content !== "string" && !(content instanceof Uint8Array)) throw new Error(`Attachment ${name} must be a string or a Uint8Array, not ${content === null ? "null" : typeof content}`);
+		if (names.has(name)) throw new Error(`Attachment ${name} was already attached`);
+		const text = typeof content === "string";
+		const original = text ? new TextEncoder().encode(content) : content;
+		const bytes = original.byteLength > maxFileBytes ? lastBytes(original, maxFileBytes, text) : original;
+		if (used + bytes.byteLength > maxRunBytes) throw new Error(`Attachment ${name} (${bytes.byteLength} bytes) would take this run past its ${maxRunBytes}-byte attachment limit (${used} used)`);
+		names.add(name);
+		used += bytes.byteLength;
+		const write = writer.writeArtifact(`attachments/${name}`, bytes);
+		pending.add(write);
+		try {
+			await write;
+			entries.push({
+				name,
+				size: bytes.byteLength,
+				...bytes.byteLength < original.byteLength ? { truncatedFrom: original.byteLength } : {}
+			});
+		} catch (error) {
+			names.delete(name);
+			used -= bytes.byteLength;
+			throw error;
+		} finally {
+			pending.delete(write);
+		}
+	}
+	return {
+		attach,
+		async close() {
+			closed = true;
+			await Promise.allSettled(pending);
+			return [...entries].sort((a, b) => a.name.localeCompare(b.name));
+		}
+	};
+}
+//#endregion
 //#region ../core/src/glob.ts
 /**
 * A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
@@ -104,9 +178,9 @@ function ignoreRules(patterns) {
 	};
 }
 /**
-* Copies the candidate workspace's regular files into the trial report,
+* Copies the candidate workspace's regular files into the run report,
 * leaving out ignored paths. Symbolic links and special files are skipped,
-* never followed, so a snapshot cannot read outside the trial. An ignored
+* never followed, so a snapshot cannot read outside the run. An ignored
 * directory is walked only when a `!` pattern could re-include something in
 * it, and is listed only if it ends up containing a snapshotted file.
 */
@@ -159,7 +233,7 @@ async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAUL
 const loads = /* @__PURE__ */ new WeakMap();
 /**
 * Loads a dataset at most once per process and project root. A failed load is
-* forgotten, so the next trial that needs the dataset tries again.
+* forgotten, so the next run that needs the dataset tries again.
 */
 function loadDataset(dataset, root, signal) {
 	let byRoot = loads.get(dataset);
@@ -210,12 +284,12 @@ async function readValueFile(id, file) {
 	}
 	throw new Error(`Dataset ${id} valueFile must be a .json or .jsonl file: ${file}`);
 }
-/** Loads (cached), derives this trial's content with `onRun`, and resolves it. */
-async function prepareDatasets(datasets, root, trial) {
+/** Loads (cached), derives this run's content with `onRun`, and resolves it. */
+async function prepareDatasets(datasets, root, run) {
 	const prepared = [];
 	for (const dataset of datasets) {
-		const loaded = await loadDataset(dataset, root, trial.signal);
-		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, trial) : loaded;
+		const loaded = await loadDataset(dataset, root, run.signal);
+		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, run) : loaded;
 		if (typeof content !== "object" || content === null) throw new Error(`Dataset ${dataset.id} onRun must return { files?, value? }`);
 		prepared.push(await resolveContent(dataset, content, root));
 	}
@@ -283,7 +357,7 @@ async function digest(files, value) {
 	if (value !== void 0) hash.update(`value ${canonicalParameters(value)}\n`);
 	return `sha256:${hash.digest("hex")}`;
 }
-function trialDatasets(prepared) {
+function runDatasets(prepared) {
 	return prepared.map(({ dataset, digest }) => ({
 		id: dataset.id,
 		visibility: dataset.visibility,
@@ -332,17 +406,17 @@ async function staticDatasetFile(dataset, root, file) {
 }
 //#endregion
 //#region ../runner/src/workspace.ts
-/** Where a trial's persistent workspace lives below `root`. */
-function trialWorkspacePath(root, trialId) {
-	return path.join(path.resolve(root), trialId.replace(/[^a-zA-Z0-9._-]/g, "_"));
+/** Where a run's persistent workspace lives below `root`. */
+function runWorkspacePath(root, runId) {
+	return path.join(path.resolve(root), runId.replace(/[^a-zA-Z0-9._-]/g, "_"));
 }
 /**
 * Creates isolated candidate and evaluator directories and writes each
 * dataset's files below its mount, on the side its visibility allows.
 * Two datasets writing the same file is an error, never an overwrite.
 */
-async function createTrialWorkspace(trialId, datasets = [], persistentRoot) {
-	const root = persistentRoot ? trialWorkspacePath(persistentRoot, trialId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
+async function createRunWorkspace(runId, datasets = [], persistentRoot) {
+	const root = persistentRoot ? runWorkspacePath(persistentRoot, runId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-run-"));
 	const persistent = persistentRoot !== void 0;
 	const artifacts = {
 		candidate: { root: path.join(root, "candidate") },
@@ -394,6 +468,20 @@ function collision(datasetId, file, cause) {
 	return cause.code === "EEXIST" ? /* @__PURE__ */ new Error(`Dataset ${datasetId} would overwrite ${file}, which another dataset already wrote`) : cause instanceof Error ? cause : new Error(String(cause));
 }
 //#endregion
+//#region ../runner/src/checkpoint-lookup.ts
+/**
+* `checkpoint(rule)` for a scoring context: the latest result of the steps
+* that are this very rule. A rule outside the transcript is a mistake in the
+* scenario, so it throws rather than reading as "not run".
+*/
+function checkpointLookup(transcript, results) {
+	return ((rule) => {
+		const steps = new Set(transcript.flatMap((step, index) => step === rule ? [index] : []));
+		if (!steps.size) throw new Error(`checkpoint(${JSON.stringify(rule.name)}) is not a checkpoint in this scenario's transcript; pass the same rule object the transcript uses`);
+		for (let index = results.length - 1; index >= 0; index--) if (steps.has(results[index].step)) return results[index];
+	});
+}
+//#endregion
 //#region ../runner/src/score.ts
 /**
 * Shared normalization for inline and final predicate and judge rules.
@@ -447,6 +535,7 @@ function runJudgeAgent(rule, agent, input, placement) {
 		yield* Effect.acquireUseRelease(Effect.tryPromise({
 			try: () => agent.start({
 				context,
+				attach: input.attach,
 				onEvent: (event) => {
 					const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
 						...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
@@ -583,7 +672,6 @@ function evaluateCheckpoint(step, index, input, now, judgeAgent) {
 	const annotations = {
 		evalId: input.context.evalId,
 		runId: input.context.runId,
-		trialId: input.context.trialId,
 		step: index,
 		checkpoint: step.name
 	};
@@ -682,7 +770,7 @@ function userMessage(step, index, { context, randomSeed }) {
 		parameters: context.parameters ?? {},
 		data: context.data,
 		seed: randomSeed,
-		trialIndex: context.trialIndex
+		...context.batch ? { batch: context.batch } : {}
 	});
 	if (typeof message !== "string" || !message.trim()) throw new Error(`The user message at step ${index + 1} must compute a non-empty string`);
 	return message;
@@ -713,7 +801,9 @@ async function runCheckpoint(definition, step, index, input) {
 		turn: progress.lastTurn,
 		...turnOutput(progress.lastTurn),
 		data: input.data,
-		checkpoints: [...progress.checkpoints]
+		checkpoints: [...progress.checkpoints],
+		checkpoint: checkpointLookup(definition.transcript, [...progress.checkpoints]),
+		attach: input.attach
 	}, now, definition.judge)).catch(async (error) => {
 		const recorded = recordError(error);
 		progress.checkpoints.push({
@@ -779,7 +869,9 @@ function scoringContext(input) {
 		...input.lastTurn ? { turn: input.lastTurn } : {},
 		...turnOutput(input.lastTurn),
 		data: input.data,
-		checkpoints: input.checkpoints
+		checkpoints: input.checkpoints,
+		checkpoint: checkpointLookup(input.transcript, input.checkpoints),
+		attach: input.attach
 	};
 }
 /** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
@@ -801,7 +893,8 @@ async function evaluateFinalScorers(definition, input) {
 			const score = await runBoundary(evaluateRule(scorer, scoringContext({
 				...input,
 				context,
-				workspace
+				workspace,
+				transcript: definition.transcript
 			}), definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
 				cause,
 				message: cause.message
@@ -841,7 +934,7 @@ async function evaluateFinalScorers(definition, input) {
 //#endregion
 //#region ../runner/src/metrics.ts
 /**
-* What EvalKit measures for every trial from its trajectory: the agent's time
+* What EvalKit measures for every run from its trajectory: the agent's time
 * (first event to the last one before final scoring), tokens when the agent
 * reported usage, and tool calls.
 */
@@ -857,8 +950,7 @@ function builtInMetrics(events) {
 	});
 	let tokens;
 	for (const event of events) if (event.source === "aut" && event.kind === "turn-completed" && event.usage) {
-		const { totalTokens, inputTokens, outputTokens } = event.usage;
-		const turn = totalTokens ?? (inputTokens === void 0 && outputTokens === void 0 ? void 0 : (inputTokens ?? 0) + (outputTokens ?? 0));
+		const turn = turnTokens(event.usage);
 		if (turn !== void 0) tokens = (tokens ?? 0) + turn;
 	}
 	if (tokens !== void 0) metrics.push({
@@ -903,7 +995,7 @@ async function evaluateMetrics(definition, context) {
 //#endregion
 //#region ../runner/src/local-report-store.ts
 const MANIFEST_FILE = "manifest.json";
-/** The project's evals and suites when the run started (schema version 4+). */
+/** The project's evals and suites when the run started. */
 const CATALOG_FILE = "catalog.json";
 async function writeJson(filePath, value) {
 	await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
@@ -922,10 +1014,12 @@ function artifactPath(root, value) {
 	if (normalized === ".." || normalized.startsWith(`..${path.sep}`)) throw new Error(`Artifact path escapes its root: ${value}`);
 	return path.join(root, normalized);
 }
-var LocalTrialWriter = class {
+var LocalRunWriter = class {
+	location;
 	directory;
 	metadata;
-	constructor(directory, metadata) {
+	constructor(location, directory, metadata) {
+		this.location = location;
 		this.directory = directory;
 		this.metadata = metadata;
 	}
@@ -933,40 +1027,13 @@ var LocalTrialWriter = class {
 		return appendFile(path.join(this.directory, "trajectory.jsonl"), `${JSON.stringify(Schema.encodeSync(TrajectoryEventSchema)(event))}\n`);
 	}
 	writeScores(scoring) {
-		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(TrialScoringSchema)(scoring));
+		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(RunScoringSchema)(scoring));
 	}
 	async writeArtifact(relativePath, data) {
 		const target = artifactPath(path.join(this.directory, "artifacts"), relativePath);
 		await mkdir(path.dirname(target), { recursive: true });
 		await writeFile(target, data);
 	}
-	async finalize(summary) {
-		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(TrialSummarySchema)(summary));
-		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
-			...Schema.encodeSync(TrialMetadataSchema)(this.metadata),
-			status: summary.status
-		});
-	}
-};
-var LocalRunWriter = class {
-	location;
-	directory;
-	metadata;
-	constructor(location, directory, metadata) {
-		this.location = location;
-		this.directory = directory;
-		this.metadata = metadata;
-	}
-	async startTrial(metadata) {
-		assertPathSegment(metadata.trialId, "trialId");
-		const directory = path.join(this.directory, "trials", metadata.trialId);
-		await mkdir(directory, { recursive: true });
-		await writeJson(path.join(directory, MANIFEST_FILE), {
-			...Schema.encodeSync(TrialMetadataSchema)(metadata),
-			status: "running"
-		});
-		return new LocalTrialWriter(directory, metadata);
-	}
 	async finalize(summary) {
 		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(RunSummarySchema)(summary));
 		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
@@ -991,17 +1058,15 @@ function localReportStore(rootDirectory) {
 }
 //#endregion
 //#region ../runner/src/local-report-reader.ts
-/** Shared v2/v3 local report boundary for CLI, dashboard, and CI consumers. */
+/** The local report boundary for CLI, dashboard, and CI consumers (schema version 6). */
 async function decode(file, schema) {
 	return Schema.decodeUnknownSync(schema)(JSON.parse(await readFile(file, "utf8")));
 }
 const readRunManifest = (root, runId) => decode(join(root, runId, "manifest.json"), RunManifestSchema);
 const readRunSummary = (root, runId) => decode(join(root, runId, "summary.json"), RunSummarySchema);
-const readTrialManifest = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "manifest.json"), TrialManifestSchema);
-const readTrialSummary = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "summary.json"), TrialSummarySchema);
-const readTrialScoring = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "scoring.json"), TrialScoringSchema);
-async function readTrialEvents(root, runId, trialId) {
-	return parseTrajectoryJsonl(await readFile(join(root, runId, "trials", trialId, "trajectory.jsonl"), "utf8"));
+const readRunScoring = (root, runId) => decode(join(root, runId, "scoring.json"), RunScoringSchema);
+async function readRunEvents(root, runId) {
+	return parseTrajectoryJsonl(await readFile(join(root, runId, "trajectory.jsonl"), "utf8"));
 }
 /**
 * True when a run's owning process is known to be gone: it ran on this host
@@ -1028,6 +1093,7 @@ function runMatrix(matrix, options) {
 		const iterator = matrix.cells(options.selection);
 		const summary = {
 			cells: 0,
+			runs: 0,
 			passed: 0,
 			failed: 0,
 			cancelled: 0
@@ -1043,7 +1109,7 @@ function runMatrix(matrix, options) {
 					"sandbox"
 				].find((name) => agentRuntimes(cell.eval.agent)[name]);
 				const suiteId = options.suiteId ?? options.suiteFor?.(cell);
-				const result = yield* runEval(cell.eval, {
+				const runs = yield* runEval(cell.eval, {
 					...options,
 					...suiteId ? { suiteId } : {},
 					concurrency: 1,
@@ -1055,18 +1121,232 @@ function runMatrix(matrix, options) {
 					}
 				});
 				summary.cells++;
-				if (result.status === "cancelled") summary.cancelled++;
-				else if (result.status === "completed" && (result.aggregateScoring?.passRate === 1 || result.scoring?.passed)) summary.passed++;
-				else summary.failed++;
-				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, result));
+				for (const run of runs) {
+					summary.runs++;
+					if (run.status === "cancelled") summary.cancelled++;
+					else if (run.status === "passed") summary.passed++;
+					else summary.failed++;
+				}
+				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, runs));
 			}
 		})), { concurrency });
 		return summary;
 	});
 }
 //#endregion
+//#region ../runner/src/upgrade.ts
+/** A new run being written. */
+const TEMPORARY = ".upgrade-";
+/** An old run set aside while its replacement, with the same ID, moves in. */
+const RETIRED = ".upgraded-";
+const PER_RUN_FILES = ["archived.json"];
+/**
+* Rewrites results from report schema versions 2 to 5 into version 6, in
+* place (RFD0008). An old run of one trial (most of them) becomes a run with
+* the same ID, so links to it keep working. An old run of several trials
+* becomes runs with the trials' IDs, in a batch with the old run's ID.
+*
+* Each new run is written under a temporary name and renamed into place;
+* artifacts are moved, not copied, as the last step before that rename. An
+* old run is removed only after its new runs exist, and one replaced under
+* the same ID is set aside first, so a crash at any point leaves the old run
+* or complete new ones, and re-running finishes the job. Version 6 runs are
+* left alone.
+*/
+async function upgradeReports(root, options = {}) {
+	const report = {
+		upgraded: 0,
+		runs: 0,
+		batches: 0,
+		running: []
+	};
+	let entries;
+	try {
+		entries = await readdir(root);
+	} catch {
+		return report;
+	}
+	if (!options.dryRun) await finishReplacements(root, entries);
+	for (const entry of entries) {
+		if (entry.startsWith(".")) continue;
+		const directory = join(root, entry);
+		const manifest = await readJson(join(directory, "manifest.json"));
+		if (!manifest || !UPGRADABLE_SCHEMA_VERSIONS.includes(manifest.schemaVersion)) continue;
+		const summary = await readJson(join(directory, "summary.json"));
+		if (!summary && !ownerIsGone(manifest.process)) {
+			report.running.push(entry);
+			continue;
+		}
+		const trials = await trialIds(directory);
+		report.upgraded += 1;
+		report.runs += Math.max(trials.length, 1);
+		if (trials.length > 1) report.batches += 1;
+		if (options.dryRun) continue;
+		await upgradeRun(root, entry, manifest, summary, trials);
+	}
+	if (!options.dryRun) await removeLeftovers(root);
+	return report;
+}
+/**
+* Finishes same-ID replacements a crash interrupted: the old run was set
+* aside, and its replacement may or may not have moved in.
+*/
+async function finishReplacements(root, entries) {
+	for (const entry of entries) {
+		if (!entry.startsWith(RETIRED)) continue;
+		const id = entry.slice(10);
+		const retired = join(root, entry);
+		const directory = join(root, id);
+		const replacement = join(root, `${TEMPORARY}${id}`);
+		if (!await exists(directory)) await rename(await exists(replacement) ? replacement : retired, directory);
+		await rm(retired, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/**
+* Temporary folders nothing will finish. One holding artifacts is kept: it
+* could be the only copy, and an old run would have moved them back.
+*/
+async function removeLeftovers(root) {
+	for (const entry of await readdir(root)) if (entry.startsWith(TEMPORARY) && !await exists(join(root, entry, "artifacts"))) await rm(join(root, entry), {
+		recursive: true,
+		force: true
+	});
+}
+/** Swaps a fully written replacement in for the old run of the same ID. */
+async function replaceInPlace(root, id, replacement) {
+	const retired = join(root, `${RETIRED}${id}`);
+	await rename(join(root, id), retired);
+	await rename(replacement, join(root, id));
+	await rm(retired, {
+		recursive: true,
+		force: true
+	});
+}
+async function upgradeRun(root, oldRunId, runManifest, runSummary, trials) {
+	const directory = join(root, oldRunId);
+	const catalog = await readJson(join(directory, "catalog.json"));
+	const extras = await presentFiles(directory, PER_RUN_FILES);
+	if (!trials.length) {
+		const target = join(root, `${TEMPORARY}${oldRunId}`);
+		await mkdir(target, { recursive: true });
+		const status = runSummary ? "cancelled" : "running";
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			runId: oldRunId,
+			status
+		}));
+		if (runSummary) await writeJsonFile(join(target, "summary.json"), {
+			status,
+			endedAt: runSummary.endedAt,
+			...runSummary.durationMs === void 0 ? {} : { durationMs: runSummary.durationMs },
+			...runSummary.error ? { error: runSummary.error } : {}
+		});
+		await copyShared(directory, target, catalog, extras);
+		await replaceInPlace(root, oldRunId, target);
+		return;
+	}
+	const inPlace = trials.length === 1;
+	for (const trialId of trials) {
+		const runId = inPlace ? oldRunId : trialId;
+		const destination = join(root, runId);
+		if (!inPlace && await exists(join(destination, "manifest.json"))) continue;
+		const source = join(directory, "trials", trialId);
+		const trialManifest = await readJson(join(source, "manifest.json")) ?? {};
+		const trialSummary = await readJson(join(source, "summary.json"));
+		const passed = (trialSummary?.scoring)?.passed;
+		const status = upgradeStatus(trialSummary?.status ?? trialManifest.status, passed);
+		const index = typeof trialManifest.trialIndex === "number" ? trialManifest.trialIndex : trials.indexOf(trialId);
+		const target = join(root, `${TEMPORARY}${runId}`);
+		if (!await exists(join(source, "artifacts")) && await exists(join(target, "artifacts"))) await rename(join(target, "artifacts"), join(source, "artifacts"));
+		await rm(target, {
+			recursive: true,
+			force: true
+		});
+		await mkdir(target, { recursive: true });
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			trial: trialManifest,
+			runId,
+			...!inPlace ? { batch: {
+				id: oldRunId,
+				index,
+				size: trials.length
+			} } : {},
+			status
+		}));
+		if (trialSummary) await writeJsonFile(join(target, "summary.json"), {
+			...trialSummary,
+			status
+		});
+		const scoring = await readText(join(source, "scoring.json"));
+		if (scoring !== void 0) await writeFile(join(target, "scoring.json"), scoring);
+		const trajectory = await readText(join(source, "trajectory.jsonl"));
+		if (trajectory !== void 0) await writeFile(join(target, "trajectory.jsonl"), upgradeTrajectory(trajectory));
+		await copyShared(directory, target, catalog, extras);
+		if (await exists(join(source, "artifacts"))) await rename(join(source, "artifacts"), join(target, "artifacts"));
+		if (inPlace) await replaceInPlace(root, oldRunId, target);
+		else await rename(target, destination);
+	}
+	if (!inPlace) await rm(directory, {
+		recursive: true,
+		force: true
+	});
+}
+async function copyShared(from, to, catalog, extras) {
+	if (catalog) await writeJsonFile(join(to, "catalog.json"), upgradeCatalog(catalog));
+	for (const file of extras) await cp(join(from, file), join(to, file));
+}
+async function trialIds(directory) {
+	try {
+		return (await readdir(join(directory, "trials"), { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
+	} catch {
+		return [];
+	}
+}
+async function presentFiles(directory, names) {
+	const present = [];
+	for (const name of names) if (await exists(join(directory, name))) present.push(name);
+	return present;
+}
+async function exists(path) {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function readText(path) {
+	try {
+		return await readFile(path, "utf8");
+	} catch {
+		return;
+	}
+}
+async function readJson(path) {
+	const text = await readText(path);
+	if (text === void 0) return void 0;
+	let value;
+	try {
+		value = JSON.parse(text);
+	} catch (error) {
+		throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+async function writeJsonFile(path, value) {
+	await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+//#endregion
 //#region ../runner/src/index.ts
-/** Effect-native aggregate eval execution entry point. */
+/**
+* Runs a scenario: once, or `repeat` times as one batch. Resolves to one
+* result per run started; runs never started because of cancellation are
+* left out.
+*/
 function runEval(definition, options) {
 	return Effect.gen(function* () {
 		yield* Effect.logInfo("eval run started").pipe(Effect.annotateLogs({
@@ -1074,19 +1354,19 @@ function runEval(definition, options) {
 			...options.runId ? { runId: options.runId } : {},
 			...options.suiteId ? { suiteId: options.suiteId } : {}
 		}));
-		const result = yield* Effect.tryPromise({
-			try: () => executeRun(definition, options),
+		const runs = yield* Effect.tryPromise({
+			try: () => executeBatch(definition, options),
 			catch: (error) => error
 		});
 		yield* Effect.logInfo("eval run finished").pipe(Effect.annotateLogs({
 			evalId: authoringId(definition),
-			runId: result.runId,
-			status: result.status
+			runs: runs.length,
+			passed: runs.filter((run) => run.status === "passed").length
 		}));
-		return result;
+		return runs;
 	});
 }
-function createId(_kind) {
+function createId() {
 	return crypto.randomUUID();
 }
 function canonicalId(kind, id) {
@@ -1095,9 +1375,9 @@ function canonicalId(kind, id) {
 function assertUuid(value, label) {
 	if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error(`${label} must be a UUID`);
 }
-function runMetadata(definition, options, runId, startedAt) {
+function runMetadata(definition, options, runId, startedAt, extra) {
 	return {
-		schemaVersion: 5,
+		schemaVersion: 6,
 		runId,
 		runUri: canonicalId("run", runId),
 		evalId: definition.id,
@@ -1110,7 +1390,9 @@ function runMetadata(definition, options, runId, startedAt) {
 		...definition.agent.identity ? { aut: definition.agent.identity } : {},
 		parameters: options.parameters,
 		matrix: options.matrix,
-		startedAt
+		startedAt,
+		...options.batch ? { batch: options.batch } : {},
+		...extra
 	};
 }
 /** Recording a secondary error must never mask the original failure or block cleanup. */
@@ -1123,104 +1405,52 @@ async function tryEmitError(error, emitRunner, now) {
 		});
 	} catch {}
 }
-/** Executes one aggregate eval run and its requested isolated trials. */
-async function executeRun(definition, options) {
+/** Starts `repeat` runs of a scenario, as one batch when there are several. */
+async function executeBatch(definition, options) {
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
 	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
+	assertNoRemovedOptions(definition);
 	if (options.runId) assertUuid(options.runId, "runId");
-	if (options.trialId) assertUuid(options.trialId, "trialId");
-	const requestedTrials = options.trials ?? definition.policy?.trials ?? 1;
-	if (!Number.isInteger(requestedTrials) || requestedTrials < 1) throw new Error(`Trial count must be a positive integer; received ${requestedTrials}`);
-	if (requestedTrials === 1) return runTrial(definition, options);
-	const now = options.now ?? (() => /* @__PURE__ */ new Date());
-	const aggregateStartedAt = now();
-	const runId = options.runId ?? createId("run");
-	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, aggregateStartedAt.toISOString()), { catalog: options.catalog });
+	const repeat = options.repeat ?? definition.repeat ?? 1;
+	if (!Number.isInteger(repeat) || repeat < 1) throw new Error(`repeat must be a positive integer; received ${repeat}`);
+	if (repeat === 1) return [await runOnce(definition, options)];
+	if (options.runId) throw new Error("runId names one run; it cannot be used with repeat");
+	const batchId = createId();
 	const signal = options.signal;
-	const trialEffects = Array.from({ length: requestedTrials }, (_, trialIndex) => {
-		const trial = Effect.tryPromise({
-			try: async () => signal?.aborted ? void 0 : runTrial(definition, {
+	const runs = Array.from({ length: repeat }, (_, index) => {
+		const run = Effect.tryPromise({
+			try: async () => signal?.aborted ? void 0 : runOnce(definition, {
 				...options,
-				runId,
-				trialId: createId("trial"),
-				trials: 1,
-				runWriter,
-				trialIndex
+				batch: {
+					id: batchId,
+					index,
+					size: repeat
+				}
 			}),
 			catch: (error) => error
 		});
-		return options.semaphore ? options.semaphore.withPermits(1)(trial) : trial;
+		return options.semaphore ? options.semaphore.withPermits(1)(run) : run;
 	});
-	const trials = (await Effect.runPromise(Effect.all(trialEffects, { concurrency: options.concurrency ?? 32 }))).filter((trial) => trial !== void 0);
-	const counts = trialCounts(trials);
-	const status = counts.cancelled || signal?.aborted ? "cancelled" : trials.some((trial) => trial.status === "failed") ? "failed" : "completed";
-	const endedAt = now();
-	await runWriter.finalize({
-		status,
-		endedAt: endedAt.toISOString(),
-		durationMs: endedAt.getTime() - aggregateStartedAt.getTime(),
-		trialCount: trials.length,
-		passed: counts.passed,
-		failed: counts.failed,
-		...counts.cancelled ? { cancelled: counts.cancelled } : {}
-	});
-	if (!trials.length) return {
-		runId,
-		runUri: canonicalId("run", runId),
-		trialId: "",
-		trialUri: canonicalId("trial", runId),
-		trialIndex: 0,
-		evalId: authoringId(definition),
-		status,
-		reportLocation: runWriter.location,
-		trialCount: 0,
-		passed: 0,
-		failed: 0,
-		aggregateScoring: {
-			passed: 0,
-			failed: 0,
-			passRate: 0
-		},
-		trials: []
-	};
-	const overallScores = trials.map((trial) => trial.scoring?.overall).filter((score) => score !== void 0);
-	return {
-		...trials[0],
-		runId,
-		evalId: authoringId(definition),
-		status,
-		trialCount: trials.length,
-		passed: counts.passed,
-		failed: counts.failed,
-		aggregateScoring: {
-			passed: counts.passed,
-			failed: counts.failed,
-			...counts.cancelled ? { cancelled: counts.cancelled } : {},
-			passRate: counts.passed / trials.length,
-			...overallScores.length ? { overall: overallScores.reduce((sum, score) => sum + score, 0) / overallScores.length } : {}
-		},
-		trials
-	};
+	return (await Effect.runPromise(Effect.all(runs, { concurrency: options.concurrency ?? 32 }))).filter((run) => run !== void 0);
 }
-async function runTrial(definition, options) {
+/** One execution: a session in its own sandbox, scored and reported. */
+async function runOnce(definition, options) {
 	const now = options.now ?? (() => /* @__PURE__ */ new Date());
-	const runId = options.runId ?? createId("run");
-	const trialId = options.trialId ?? createId("trial");
+	const runId = options.runId ?? createId();
 	const startedAt = now().toISOString();
 	const selectedRuntime = options.runtime;
 	const runtime = selectedRuntime ? agentRuntimes(definition.agent)[selectedRuntime] : void 0;
 	if (selectedRuntime && !runtime) throw new Error(`AUT does not declare the requested ${selectedRuntime} runtime`);
 	const randomSeed = crypto.getRandomValues(/* @__PURE__ */ new Uint32Array(1))[0];
-	const trialIndex = options.trialIndex ?? 0;
+	const batch = options.batch;
 	const signal = options.signal ?? new AbortController().signal;
 	const root = options.root ?? process.cwd();
 	const baseContext = {
 		runId,
 		evalId: authoringId(definition),
-		trialId,
-		trialIndex,
+		...batch ? { batch } : {},
 		metadata: {
 			...toJsonObject(definition.metadata ?? {}, `${definition.id}.metadata`),
 			randomSeed
@@ -1234,8 +1464,7 @@ async function runTrial(definition, options) {
 		prepared = await prepareDatasets(definition.datasets ?? [], root, {
 			evalId: baseContext.evalId,
 			runId,
-			trialId,
-			trialIndex,
+			...batch ? { batch } : {},
 			seed: randomSeed,
 			parameters: options.parameters ?? {},
 			signal
@@ -1247,40 +1476,28 @@ async function runTrial(definition, options) {
 	const scoringData = datasetValues(prepared, "all");
 	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
 	const events = [];
-	const runWriter = options.runWriter ?? await options.report.startRun(runMetadata(definition, options, runId, startedAt), { catalog: options.catalog });
-	const trialWriter = await runWriter.startTrial({
-		schemaVersion: 5,
-		runId,
-		runUri: canonicalId("run", runId),
-		trialId,
-		trialUri: canonicalId("trial", trialId),
-		trialIndex: options.trialIndex ?? 0,
-		evalId: definition.id,
-		...definition.agent.identity ? { aut: definition.agent.identity } : {},
-		parameters: options.parameters,
-		matrix: options.matrix,
-		startedAt,
+	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, startedAt, {
 		workspaceIgnore,
-		...prepared.length ? { datasets: trialDatasets(prepared) } : {}
-	});
+		...prepared.length ? { datasets: runDatasets(prepared) } : {}
+	}), { catalog: options.catalog });
+	const attachments = attachmentStore(runWriter, options.attachmentLimits);
 	let writeChain = Promise.resolve();
 	let reportingFailed = false;
-	const trialRef = {
+	const runRef = {
 		runId,
-		trialId,
-		trialIndex,
+		...batch ? { batch } : {},
 		evalId: definition.id,
 		...options.suiteId ? { suiteId: options.suiteId } : {},
 		...options.parameters ? { parameters: options.parameters } : {}
 	};
 	const observe = (event) => {
 		try {
-			options.onEvent?.(trialRef, event);
+			options.onEvent?.(runRef, event);
 		} catch {}
 	};
 	const append = async (event) => {
 		events.push(event);
-		writeChain = writeChain.then(() => trialWriter.appendEvent(event));
+		writeChain = writeChain.then(() => runWriter.appendEvent(event));
 		try {
 			await writeChain;
 			observe(event);
@@ -1316,7 +1533,7 @@ async function runTrial(definition, options) {
 	const workspaceScope = await Effect.runPromise(Scope.make());
 	try {
 		await emitRunner({
-			kind: "trial-started",
+			kind: "run-started",
 			timestamp: now().toISOString()
 		});
 		if (signal.aborted) throw cancelled();
@@ -1325,7 +1542,7 @@ async function runTrial(definition, options) {
 			message: datasetsError instanceof Error ? datasetsError.message : String(datasetsError)
 		});
 		workspace = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
-			try: () => createTrialWorkspace(trialId, prepared, options.workspaceRoot),
+			try: () => createRunWorkspace(runId, prepared, options.workspaceRoot),
 			catch: (cause) => new DatasetError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
@@ -1338,20 +1555,21 @@ async function runTrial(definition, options) {
 			})
 		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
 			cleanupError = error;
-		}).pipe(Effect.tap(() => Effect.logError("trial workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
-		const trialWorkspace = workspace;
-		const trialContext = {
+		}).pipe(Effect.tap(() => Effect.logError("run workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
+		const runWorkspace = workspace;
+		const runContext = {
 			...baseContext,
 			data: candidateData,
-			workspace: trialWorkspace.artifacts.candidate,
-			evaluatorWorkspace: trialWorkspace.artifacts.evaluator
+			workspace: runWorkspace.artifacts.candidate,
+			evaluatorWorkspace: runWorkspace.artifacts.evaluator
 		};
-		context = trialContext;
+		context = runContext;
 		await runBoundary(Effect.acquireUseRelease(Effect.tryPromise({
 			try: () => definition.agent.start({
-				context: trialContext,
+				context: runContext,
 				...runtime ? { runtime } : {},
-				onEvent: onAutEvent
+				onEvent: onAutEvent,
+				attach: attachments.attach
 			}),
 			catch: (cause) => new AutExecutionError({
 				cause,
@@ -1359,8 +1577,8 @@ async function runTrial(definition, options) {
 			})
 		}), (session) => Effect.tryPromise({
 			try: () => executeTranscript(definition, session, {
-				context: trialContext,
-				artifacts: trialWorkspace.artifacts,
+				context: runContext,
+				artifacts: runWorkspace.artifacts,
 				events,
 				now,
 				randomSeed,
@@ -1368,7 +1586,8 @@ async function runTrial(definition, options) {
 				emitAut,
 				data: scoringData,
 				signal,
-				progress
+				progress,
+				attach: attachments.attach
 			}),
 			catch: (cause) => cause
 		}), (opened) => Effect.tryPromise({
@@ -1381,7 +1600,6 @@ async function runTrial(definition, options) {
 			closeError = error;
 		}).pipe(Effect.tap(() => Effect.logError("AUT session close failed")))))).pipe(Effect.annotateLogs({
 			runId,
-			trialId,
 			evalId: definition.id
 		})));
 	} catch (error) {
@@ -1411,6 +1629,7 @@ async function runTrial(definition, options) {
 			skippedScorers,
 			data: scoringData,
 			checkpoints,
+			attach: attachments.attach,
 			now,
 			emitRunner
 		});
@@ -1421,12 +1640,14 @@ async function runTrial(definition, options) {
 			events,
 			lastTurn,
 			data: scoringData,
-			checkpoints
+			checkpoints,
+			transcript: definition.transcript,
+			attach: attachments.attach
 		})));
 		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
-		await trialWriter.writeScores(scoring);
+		await runWriter.writeScores(scoring);
 		if (workspace) try {
-			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter, workspaceIgnore);
+			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, runWriter, workspaceIgnore);
 		} catch (captureError) {
 			if (!primaryError) primaryError = captureError;
 			await tryEmitError(captureError, emitRunner, now);
@@ -1444,49 +1665,34 @@ async function runTrial(definition, options) {
 	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
 	if (!primaryError) try {
 		await emitRunner({
-			kind: "trial-completed",
+			kind: "run-completed",
 			timestamp: now().toISOString()
 		});
 	} catch (error) {
 		primaryError = error;
 	}
-	status = wasCancelled ? "cancelled" : primaryError ? "failed" : "completed";
+	status = wasCancelled ? "cancelled" : primaryError ? "errored" : scoring.passed ? "passed" : "failed";
 	const error = primaryError ? recordError(primaryError) : void 0;
+	const attached = await attachments.close();
 	const endedAt = now().toISOString();
-	await trialWriter.finalize({
+	const durationMs = new Date(endedAt).getTime() - new Date(startedAt).getTime();
+	await runWriter.finalize({
 		status,
 		endedAt,
-		durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
+		durationMs,
 		scoring,
 		...artifacts ? { artifacts } : {},
+		...attached.length ? { attachments: attached } : {},
 		...error ? { error } : {}
 	});
-	if (!options.runWriter) {
-		const counts = trialCounts([{
-			status,
-			scoring
-		}]);
-		await runWriter.finalize({
-			status,
-			endedAt,
-			durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
-			trialCount: 1,
-			passed: counts.passed,
-			failed: counts.failed,
-			...counts.cancelled ? { cancelled: counts.cancelled } : {},
-			...error ? { error } : {}
-		});
-	}
 	return {
 		runId,
 		runUri: canonicalId("run", runId),
-		trialId,
-		trialUri: canonicalId("trial", trialId),
-		trialIndex: options.trialIndex ?? 0,
 		evalId: definition.id,
 		status,
+		...batch ? { batch } : {},
 		reportLocation: runWriter.location,
-		durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
+		durationMs,
 		scoring,
 		...error ? { error } : {}
 	};
@@ -1500,23 +1706,9 @@ function datasetValues(prepared, audience) {
 	}
 	return values;
 }
-/** A cancelled trial is neither passed nor failed. */
-function trialCounts(trials) {
-	let passed = 0;
-	let failed = 0;
-	let cancelledTrials = 0;
-	for (const trial of trials) if (trial.status === "cancelled") cancelledTrials++;
-	else if (trial.status === "completed" && trial.scoring?.passed) passed++;
-	else failed++;
-	return {
-		passed,
-		failed,
-		cancelled: cancelledTrials
-	};
-}
-/** Persistent workspaces of trials that did not pass are kept for debugging by default. */
+/** Persistent workspaces of runs that did not pass are kept for debugging by default. */
 function keepWorkspace(policy = "failed", passed) {
 	return policy === "all" || policy === "failed" && !passed;
 }
 //#endregion
-export { CheckpointExecutionError as _, readRunSummary as a, ScoringError as b, readTrialScoring as c, trialWorkspacePath as d, staticDatasetFile as f, CancelledError as g, AutExecutionError as h, readRunManifest as i, readTrialSummary as l, globMatcher as m, runMatrix as n, readTrialEvents as o, staticDatasetFiles as p, ownerIsGone as r, readTrialManifest as s, runEval as t, localReportStore as u, DatasetError as v, ReportError as y };
+export { DatasetError as _, readRunEvents as a, readRunSummary as c, staticDatasetFile as d, staticDatasetFiles as f, CheckpointExecutionError as g, CancelledError as h, ownerIsGone as i, localReportStore as l, AutExecutionError as m, upgradeReports as n, readRunManifest as o, globMatcher as p, runMatrix as r, readRunScoring as s, runEval as t, runWorkspacePath as u, ReportError as v, ScoringError as y };