@leostera/evalkit 0.8.0 → 0.10.0

package/dist/cli.mjs

@@ -1,12 +1,14 @@
 #!/bin/sh
 ':' //; exec "${EVALKIT_RUNTIME:-$(command -v bun || command -v node)}" "$0" "$@"
-import { B as defineEvalMatrix, V as canonicalParameters, a as authoringId, c as defineSuite, i as agentRuntimes, m as registerEvals, p as recordError } from "./src-LeGSyDD2.mjs";
-import { a as readRunSummary, d as trialWorkspacePath, f as staticDatasetFile, i as readRunManifest, l as readTrialSummary, m as globMatcher, n as runMatrix, o as readTrialEvents, p as staticDatasetFiles, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore } from "./src-mm8L_IxM.mjs";
-import { c as dashboardIndex, d as encodeBundle, o as repoKey } from "./index-db-vbqEGFaU.mjs";
-import { t as createDirectoryShareApp } from "./src-CZnDXP1K.mjs";
+import { H as defineEvalMatrix, U as canonicalParameters, h as registerEvals, i as agentRuntimes, l as defineSuite, m as recordError, o as authoringId, tt as passMark } from "./src-B14fXAGp.mjs";
+import { a as readRunEvents, c as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, i as ownerIsGone, l as localReportStore, n as upgradeReports, o as readRunManifest, p as globMatcher, r as runMatrix, t as runEval, u as runWorkspacePath } from "./src-BfSQKOJP.mjs";
+import { f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, s as repoKey } from "./index-db-B1FeXzmS.mjs";
+import { t as createDirectoryShareApp } from "./src-Bccv9igQ.mjs";
 import { Effect, LogLevel, Logger } from "effect";
-import { cp, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
-import path, { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
+import { tmpdir } from "node:os";
+import { cp, lstat, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
+import path, { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { createHash } from "node:crypto";
 import fs, { existsSync } from "node:fs";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { Hono } from "hono";
@@ -166,63 +168,18 @@ function shortSource(source) {
 //#endregion
 //#region ../cli/src/dashboard-reports.ts
 const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-function runMetrics(trials) {
-	const byName = /* @__PURE__ */ new Map();
-	for (const trial of trials) for (const result of trial?.scoring?.metrics ?? []) {
-		const entry = byName.get(result.name) ?? {
-			name: result.name,
-			...result.unit ? { unit: result.unit } : {},
-			...result.better ? { better: result.better } : {},
-			...result.builtIn ? { builtIn: true } : {},
-			count: 0,
-			sum: 0
-		};
-		if (result.value !== void 0) {
-			entry.count += 1;
-			entry.sum += result.value;
-		}
-		byName.set(result.name, entry);
-	}
-	return [...byName.values()].map(({ sum, ...metric }) => ({
-		...metric,
-		...metric.count ? { mean: sum / metric.count } : {}
-	}));
-}
-function runScorers(trials) {
-	const byKey = /* @__PURE__ */ new Map();
-	for (const trial of trials) for (const score of trial?.scoring?.results ?? []) {
-		const kind = score.kind;
-		const key = `${kind}\0${score.name}`;
-		const entry = byKey.get(key) ?? {
-			name: score.name,
-			kind,
-			count: 0,
-			skipped: 0,
-			sum: 0
-		};
-		if (score.skipped) entry.skipped += 1;
-		else if (score.value !== void 0) {
-			entry.count += 1;
-			entry.sum += score.value;
-		}
-		byKey.set(key, entry);
-	}
-	return [...byKey.values()].map(({ sum, ...scorer }) => ({
-		...scorer,
-		...scorer.count ? { mean: sum / scorer.count } : {}
-	}));
-}
-function runDatasets(manifests) {
-	const byId = /* @__PURE__ */ new Map();
-	for (const manifest of manifests) for (const dataset of manifest?.datasets ?? []) {
-		const digests = byId.get(dataset.id) ?? /* @__PURE__ */ new Set();
-		digests.add(dataset.digest);
-		byId.set(dataset.id, digests);
-	}
-	return [...byId].map(([id, digests]) => ({
-		id,
-		digests: [...digests].sort()
-	}));
+const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]|\u001b\][^\u0007]*(?:\u0007|\u001b\\)|\u001b[@-Z\\-_]/g;
+const CONTROL = /[\u0000-\u0008\u000b-\u001f\u007f]/g;
+/**
+* The first non-empty line of a message, without terminal colours, cut to
+* `max` characters: a scorer may explain itself with 300 lines of stderr,
+* and a summary line must stay one line. The run page has the rest.
+*/
+function firstLine(text, max = 100) {
+	const lines = (text ?? "").replace(ANSI, "").split(/\r?\n/).map((line) => line.replace(CONTROL, "").trim()).filter(Boolean);
+	const line = lines[0] ?? "";
+	const more = lines.length > 1 ? " …" : "";
+	return line.length > max ? `${line.slice(0, max - 1)}…` : `${line}${more}`;
 }
 /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
 const ARCHIVE_MARKER = "archived.json";
@@ -240,23 +197,19 @@ async function ifPresent(operation) {
 function assertUuid(id, label) {
 	if (!UUID_PATTERN.test(id)) throw new Error(`Invalid ${label}`);
 }
-function trialPath(root, runId, trialId) {
+function runPath(root, runId) {
 	assertUuid(runId, "run ID");
-	assertUuid(trialId, "trial ID");
-	return resolve(root, runId, "trials", trialId);
-}
-function runStatus(summary) {
-	if (summary.status === "running") return "running";
-	if (summary.status === "cancelled") return "cancelled";
-	if (summary.status !== "completed") return "errored";
-	return summary.failed === 0 && summary.passed === summary.trialCount ? "passed" : "failed";
-}
-function trialStatus(summary) {
-	if (summary.status === "running") return "running";
-	if (summary.status === "cancelled") return "cancelled";
-	if (summary.status !== "completed") return "errored";
-	return summary.scoring?.passed === true ? "passed" : "failed";
+	return resolve(root, runId);
 }
+/**
+* Headers for files an agent, a dataset or an adapter produced: untrusted
+* content on the dashboard's origin, so never rendered or sniffed as HTML.
+*/
+const UNTRUSTED_FILE = {
+	"content-type": "application/octet-stream",
+	"x-content-type-options": "nosniff",
+	"content-security-policy": "sandbox"
+};
 function fileContentType(file) {
 	return {
 		".css": "text/css",
@@ -265,15 +218,17 @@ function fileContentType(file) {
 	}[extname(file)] ?? "application/octet-stream";
 }
 /**
-* Read saved local reports. A missing summary means "running"; malformed reports are errors.
-* Runs written by an older EvalKit (manifest `schemaVersion` < 2) are skipped and counted.
+* Read saved local reports. A missing summary means "running"; malformed
+* reports are errors. Runs from an older report format (before schema
+* version 6) are skipped and counted, for `evalkit upgrade` to convert.
 */
-function createDashboardReportReader(root, registry) {
+function createDashboardReportReader(root) {
 	const legacyRuns = /* @__PURE__ */ new Set();
 	async function listRuns() {
 		const entries = await ifPresent(() => readdir(root, { withFileTypes: true }));
 		if (!entries) return [];
 		const ids = entries.filter((entry) => entry.isDirectory() && UUID_PATTERN.test(entry.name)).map((entry) => entry.name);
+		legacyRuns.clear();
 		return (await Promise.all(ids.map(readRun))).filter((run) => run !== void 0);
 	}
 	/** True for a run from a report format this version cannot read; never for corrupt JSON. */
@@ -286,7 +241,7 @@ function createDashboardReportReader(root, registry) {
 		} catch {
 			return false;
 		}
-		return typeof version === "number" && version < 2;
+		return typeof version === "number" && version < 6;
 	}
 	async function readRun(id) {
 		if (await isLegacyRun(id)) {
@@ -296,15 +251,9 @@ function createDashboardReportReader(root, registry) {
 		const manifest = await ifPresent(() => readRunManifest(root, id));
 		if (!manifest) return void 0;
 		const summary = await ifPresent(() => readRunSummary(root, id));
-		const trialIds = await ifPresent(() => readdir(resolve(root, id, "trials")));
-		const trials = await Promise.all((trialIds ?? []).map((trialId) => ifPresent(() => readTrialSummary(root, id, trialId))));
-		const trialManifests = await Promise.all((trialIds ?? []).map((trialId) => ifPresent(() => readTrialManifest(root, id, trialId))));
-		const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
-		const scorers = runScorers(trials);
-		const metrics = runMetrics(trials);
-		const datasets = runDatasets(trialManifests);
 		const archived = await ifPresent(() => stat(resolve(root, id, ARCHIVE_MARKER)));
 		const interrupted = !summary && ownerIsGone(manifest.process);
+		const scoring = summary?.scoring;
 		return {
 			id,
 			...archived ? { archived: true } : {},
@@ -321,76 +270,81 @@ function createDashboardReportReader(root, registry) {
 				manifest.aut.id,
 				manifest.aut.version
 			].filter(Boolean).join(" / ") } : {},
-			status: summary ? runStatus(summary) : interrupted ? "cancelled" : "running",
+			status: summary ? summary.status : interrupted ? "cancelled" : "running",
 			...interrupted ? { interrupted: true } : {},
 			startedAt: manifest.startedAt,
 			...summary?.endedAt ? { completedAt: summary.endedAt } : {},
 			...summary?.durationMs !== void 0 ? { durationMs: summary.durationMs } : {},
-			completedTrials: summary?.trialCount ?? trials.filter(Boolean).length,
-			requestedTrials: summary?.trialCount ?? registry.get(manifest.evalId)?.policy?.trials ?? 1,
-			...scores.length ? { score: scores.reduce((sum, score) => sum + score, 0) / scores.length } : {},
-			...scorers.length ? { scorers } : {},
-			...metrics.length ? { metrics } : {},
-			...datasets.length ? { datasets } : {}
-		};
-	}
-	async function listTrials(runId) {
-		assertUuid(runId, "run ID");
-		const ids = await ifPresent(() => readdir(resolve(root, runId, "trials")));
-		if (!ids) return [];
-		const runManifest = await ifPresent(() => readRunManifest(root, runId));
-		const ownerGone = ownerIsGone(runManifest?.process);
-		return (await Promise.all(ids.map(async (id) => {
-			const manifest = await readTrialManifest(root, runId, id);
-			const summary = await ifPresent(() => readTrialSummary(root, runId, id));
-			return {
+			...manifest.batch ? { batch: manifest.batch } : {},
+			...scoring?.overall === void 0 ? {} : { score: scoring.overall },
+			...scoring?.results.length ? { scorers: scoring.results.map((result) => ({
+				name: result.name,
+				kind: result.kind,
+				...result.skipped ? { skipped: true } : {},
+				...result.value === void 0 || result.skipped ? {} : { value: result.value },
+				...result.passed === void 0 || result.skipped ? {} : { passed: result.passed },
+				...result.error ? { errored: true } : {},
+				...result.choice ? { choice: result.choice } : {},
+				...result.explanation ? { explanation: firstLine(result.explanation, 200) } : {},
+				...result.judge?.model ? { model: result.judge.model } : {}
+			})) } : {},
+			...scoring?.metrics?.length ? { metrics: scoring.metrics.map((metric) => ({
+				name: metric.name,
+				...metric.unit ? { unit: metric.unit } : {},
+				...metric.better ? { better: metric.better } : {},
+				...metric.builtIn ? { builtIn: true } : {},
+				...metric.value === void 0 ? {} : { value: metric.value }
+			})) } : {},
+			...manifest.datasets?.length ? { datasets: manifest.datasets.map(({ id, digest }) => ({
 				id,
-				index: manifest.trialIndex,
-				status: summary ? trialStatus(summary) : ownerGone ? "cancelled" : "running",
-				startedAt: manifest.startedAt,
-				...summary?.endedAt ? { completedAt: summary.endedAt } : {},
-				...summary?.durationMs !== void 0 ? { durationMs: summary.durationMs } : {},
-				...summary?.scoring?.overall === void 0 ? {} : { score: summary.scoring.overall },
-				scores: summary?.scoring?.results ?? [],
-				...summary?.scoring?.metrics ? { metrics: summary.scoring.metrics } : {},
-				checkpoints: summary?.scoring?.checkpoints ?? [],
-				...summary?.scoring?.skippedScorers ? { skippedScorers: summary.scoring.skippedScorers } : {}
-			};
-		}))).sort((left, right) => left.index - right.index);
+				digest
+			})) } : {}
+		};
 	}
-	async function trialDetail(runId, trialId) {
-		trialPath(root, runId, trialId);
+	async function runDetail(runId) {
+		runPath(root, runId);
+		const manifest = await readRunManifest(root, runId);
+		const summary = await ifPresent(() => readRunSummary(root, runId));
+		const scoring = summary?.scoring;
 		return {
-			manifest: await readTrialManifest(root, runId, trialId),
-			summary: await ifPresent(() => readTrialSummary(root, runId, trialId)) ?? { status: "running" }
+			scores: scoring?.results ?? [],
+			...scoring?.metrics ? { metrics: scoring.metrics } : {},
+			checkpoints: scoring?.checkpoints ?? [],
+			...scoring?.skippedScorers ? { skippedScorers: scoring.skippedScorers } : {},
+			...summary?.attachments?.length ? { attachments: summary.attachments } : {},
+			manifest,
+			summary: summary ?? { status: "running" }
 		};
 	}
-	async function listArtifacts(runId, trialId) {
-		return walkArtifacts(resolve(trialPath(root, runId, trialId), "artifacts"));
+	async function listArtifacts(runId) {
+		return walkArtifacts(resolve(runPath(root, runId), "artifacts"));
 	}
-	async function listCandidateWorkspace(runId, trialId) {
-		return walkArtifacts(resolve(trialPath(root, runId, trialId), "artifacts", "candidate"));
+	async function listCandidateWorkspace(runId) {
+		return walkArtifacts(resolve(runPath(root, runId), "artifacts", "candidate"));
 	}
-	async function candidateFile(runId, trialId, relativePath) {
-		const candidateRoot = resolve(trialPath(root, runId, trialId), "artifacts", "candidate");
-		const file = resolve(candidateRoot, relativePath);
-		if (file !== candidateRoot && !file.startsWith(`${candidateRoot}${sep}`)) throw new Error("Invalid workspace path");
-		if (!(await stat(file)).isFile()) throw new Error("Workspace path is not a file");
-		return new Response(await readFile(file), { headers: { "content-type": fileContentType(file) } });
+	/** A file below one of a run's artifact trees; never outside it. */
+	async function artifactFile(runId, tree, relativePath) {
+		const treeRoot = resolve(runPath(root, runId), "artifacts", tree);
+		const file = resolve(treeRoot, relativePath);
+		if (file === treeRoot || !file.startsWith(`${treeRoot}${sep}`)) throw new Error(`Invalid ${tree === "candidate" ? "workspace" : "attachment"} path`);
+		if (!(await lstat(file)).isFile()) throw new Error(`${tree === "candidate" ? "Workspace path" : "Attachment"} is not a file`);
+		return new Response(await readFile(file), { headers: UNTRUSTED_FILE });
 	}
-	async function trajectory(runId, trialId) {
-		trialPath(root, runId, trialId);
-		return await ifPresent(() => readTrialEvents(root, runId, trialId)) ?? [];
+	const candidateFile = (runId, path) => artifactFile(runId, "candidate", path);
+	const attachmentFile = (runId, name) => artifactFile(runId, "attachments", name);
+	async function trajectory(runId) {
+		runPath(root, runId);
+		return await ifPresent(() => readRunEvents(root, runId)) ?? [];
 	}
 	return {
 		listRuns,
-		/** Run IDs skipped by the last `listRuns()` because an older EvalKit wrote them. */
+		/** Run IDs skipped by the last `listRuns()`: an older EvalKit wrote them; `evalkit upgrade` converts them. */
 		legacyRunIds: () => [...legacyRuns],
-		listTrials,
-		trialDetail,
+		runDetail,
 		listArtifacts,
 		listCandidateWorkspace,
 		candidateFile,
+		attachmentFile,
 		trajectory
 	};
 }
@@ -461,40 +415,32 @@ async function buildDashboard(options, step = quietStep) {
 		}
 		return files;
 	}, (files) => count(files, "dataset file"));
-	const reports = createDashboardReportReader(reportRoot, registry);
+	const reports = createDashboardReportReader(reportRoot);
 	const snapshot = await step("Run reports", async (progress) => {
 		const all = await reports.listRuns();
 		const runs = all.filter((run) => !run.archived);
 		await write("runs.json", { runs });
-		let trials = 0;
 		let files = 0;
-		const runTrials = [];
 		for (const [index, run] of runs.entries()) {
 			progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
 			const written = await snapshotRun(reports, run.id, directorySink(resolve(data, "runs", run.id)));
-			trials += written.trialIds.length;
 			files += written.files;
-			runTrials.push({
-				id: run.id,
-				trials: written.trialIds
-			});
 		}
 		return {
 			runs: runs.length,
-			trials,
 			files,
 			skipped: reports.legacyRunIds().length,
 			archived: all.length - runs.length,
-			runTrials
+			runIds: runs.map((run) => run.id)
 		};
-	}, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
+	}, ({ runs, files, skipped, archived }) => `${count(runs, "run")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped (run \`evalkit upgrade\`)` : ""));
 	await step("Route pages", async () => {
 		const catalog = registry.catalog();
 		const routes = staticRoutes({
 			scenarios: catalog.map((entry) => entry.id),
 			agents: catalog.flatMap((entry) => entry.agent.id ? [entry.agent.id] : []),
 			datasets: catalog.flatMap((entry) => entry.datasets.map((dataset) => dataset.id)),
-			runs: snapshot.runTrials
+			runs: snapshot.runIds
 		});
 		for (const route of routes) {
 			const file = resolve(outDir, `${route}.html`);
@@ -504,7 +450,7 @@ async function buildDashboard(options, step = quietStep) {
 		await writeFile(resolve(outDir, "404.html"), dashboardIndex(template, { base: "/" }));
 		return routes.length;
 	}, (pages) => `${count(pages, "page")} with clean /path URLs`);
-	const { runTrials: _runs, ...summary } = snapshot;
+	const { runIds: _runs, ...summary } = snapshot;
 	return {
 		outDir,
 		...summary
@@ -522,7 +468,6 @@ function staticRoutes(ids) {
 	return [
 		"scoreboard",
 		"runs",
-		"trials",
 		"workspaces",
 		"library/agents",
 		"library/judges",
@@ -531,9 +476,22 @@ function staticRoutes(ids) {
 		...under("library/scenarios", ids.scenarios),
 		...under("library/agents", ids.agents),
 		...under("library/datasets", ids.datasets),
-		...ids.runs.filter((run) => safe(run.id)).flatMap((run) => [`runs/${run.id}`, ...run.trials.filter(safe).flatMap((trial) => [`runs/${run.id}/trials/${trial}`, `runs/${run.id}/trials/${trial}/workspace`])])
+		...ids.runs.filter(safe).flatMap((id) => [`runs/${id}`, ...RUN_SECTIONS.map((section) => `runs/${id}/${section}`)])
 	];
 }
+/**
+* A run's sections, each its own page; `attachments` is the logs' URL from
+* before 0.10, kept so old links redirect.
+*/
+const RUN_SECTIONS = [
+	"judging",
+	"metrics",
+	"workspace",
+	"logs",
+	"datasets",
+	"raw",
+	"attachments"
+];
 /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
 function baseFor(route) {
 	const depth = route.split("/").length - 1;
@@ -564,33 +522,33 @@ function directorySink(root) {
 }
 /**
 * One run's snapshot, shared by `evalkit build` (files on disk) and
-* `evalkit share` (an upload bundle): `trials.json`, then per trial
-* `detail.json`, `events.json`, `artifacts.json`, `workspace.json`, and the
-* candidate files under `workspace/`. Evaluator workspaces are never read.
+* `evalkit share` (an upload bundle): `detail.json`, `events.json`,
+* `artifacts.json` and `workspace.json`, the candidate files under
+* `workspace/`, and attachments under `attachments/` (both left out with
+* `workspaceFiles: false`; they stay listed). Evaluator workspaces are never
+* read.
 */
 async function snapshotRun(reports, runId, sink, options = {}) {
-	const trials = await reports.listTrials(runId);
-	await sink.json("trials.json", { trials });
+	const detail = await reports.runDetail(runId);
+	await sink.json("detail.json", detail);
+	await sink.json("events.json", { events: await reports.trajectory(runId) });
+	await sink.json("artifacts.json", { artifacts: await reports.listArtifacts(runId) });
+	const workspace = await reports.listCandidateWorkspace(runId);
+	await sink.json("workspace.json", workspace);
+	if (options.workspaceFiles === false) return { files: 0 };
 	let files = 0;
-	for (const trial of trials) {
-		const base = `trials/${trial.id}`;
-		await sink.json(`${base}/detail.json`, await reports.trialDetail(runId, trial.id));
-		await sink.json(`${base}/events.json`, { events: await reports.trajectory(runId, trial.id) });
-		await sink.json(`${base}/artifacts.json`, { artifacts: await reports.listArtifacts(runId, trial.id) });
-		const workspace = await reports.listCandidateWorkspace(runId, trial.id);
-		await sink.json(`${base}/workspace.json`, workspace);
-		if (options.workspaceFiles === false) continue;
-		for (const entry of workspace) {
-			if (entry.kind !== "file") continue;
-			const response = await reports.candidateFile(runId, trial.id, entry.path);
-			await sink.bytes(`${base}/workspace/${entry.path}`, new Uint8Array(await response.arrayBuffer()));
-			files += 1;
-		}
+	for (const attachment of detail.attachments ?? []) {
+		const response = await reports.attachmentFile(runId, attachment.name);
+		await sink.bytes(`attachments/${attachment.name}`, new Uint8Array(await response.arrayBuffer()));
+		files += 1;
 	}
-	return {
-		trialIds: trials.map((trial) => trial.id),
-		files
-	};
+	for (const entry of workspace) {
+		if (entry.kind !== "file") continue;
+		const response = await reports.candidateFile(runId, entry.path);
+		await sink.bytes(`workspace/${entry.path}`, new Uint8Array(await response.arrayBuffer()));
+		files += 1;
+	}
+	return { files };
 }
 /** Total size and file count of a directory, for the build summary. */
 async function directorySize(root) {
@@ -639,7 +597,7 @@ function listMatrixCells(matrix, input) {
 	if (![
 		"eval",
 		"agent",
-		"trials",
+		"repeat",
 		...[...Object.keys(matrix.cases?.[0] ?? {}), ...Object.keys(matrix.parameters)].map((key) => `axis:${key}`)
 	].includes(input.sort)) throw new Error(`Unknown matrix sort: ${input.sort}`);
 	const selection = input.evalIds.length ? { evals: input.evalIds } : {};
@@ -663,7 +621,7 @@ function listMatrixCells(matrix, input) {
 		}
 		if (input.sort === "eval") return cell.eval.name ?? cell.eval.id;
 		if (input.sort === "agent") return cell.eval.agent?.identity?.name ?? cell.eval.agent?.identity?.kind ?? "";
-		return cell.eval.policy?.trials ?? 1;
+		return cell.eval.repeat ?? 1;
 	}
 	cells.sort((left, right) => {
 		const a = value(left);
@@ -894,7 +852,12 @@ async function discover(root, config) {
 async function isDirectory(path) {
 	return stat(path).then((info) => info.isDirectory(), () => false);
 }
-async function loadProject(cwd, configPath) {
+/**
+* The project's config and root, without importing any scenario: enough for
+* commands that only touch results (`evalkit upgrade`), and that must work
+* while scenarios from an older EvalKit still fail to load.
+*/
+async function loadConfig(cwd, configPath) {
 	const candidates = configPath ? [resolve(cwd, configPath)] : [
 		"evalkit.config.ts",
 		"evalkit.config.js",
@@ -908,7 +871,17 @@ async function loadProject(cwd, configPath) {
 	const root = file ? dirname(file) : resolve(cwd);
 	const config = file ? (await importProjectModule(file)).default : {};
 	if (!config || typeof config !== "object" || Array.isArray(config)) throw new Error("Config must default-export defineConfig({...})");
+	if (config.execution && "trials" in config.execution) throw new Error("execution.trials was renamed to execution.repeat in EvalKit 0.9");
+	if (config.judge !== void 0 && typeof config.judge?.start !== "function") throw new Error("judge in the config must be an agent made with defineAgent()");
 	for (const [key, value] of Object.entries(config.execution ?? {})) if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) throw new Error(`execution.${key} must be a positive integer`);
+	return {
+		root,
+		...file ? { file } : {},
+		config
+	};
+}
+async function loadProject(cwd, configPath) {
+	const { root, file, config } = await loadConfig(cwd, configPath);
 	let registry = config.registry;
 	if (!file && !configPath && await exists$1(resolve(root, "src/registry.ts"))) registry = (await importProjectModule(resolve(root, "src/registry.ts"))).default;
 	let loadErrors = [];
@@ -937,6 +910,17 @@ async function loadProject(cwd, configPath) {
 			matrices: [...registry.matrices, matrix]
 		};
 	}
+	const judge = config.judge;
+	if (judge) {
+		const catalog = registry.catalog;
+		registry = {
+			...registry,
+			catalog: (options) => catalog({
+				judge,
+				...options
+			})
+		};
+	}
 	return {
 		root,
 		config,
@@ -944,6 +928,34 @@ async function loadProject(cwd, configPath) {
 		loadErrors
 	};
 }
+/**
+* Where a project's run workspaces live. By default outside the project, in
+* the system temp directory (one directory per project, so `tidy` and `dev`
+* find the ones it kept): an agent that looks around its workspace finds no
+* project source, no `AGENTS.md` or skills above it, and no results.
+* `sandboxDir` overrides it; `inside` says when that puts them back in the
+* project, which `run` warns about.
+*/
+function sandboxLocation(project) {
+	if (project.config.sandboxDir === void 0) return {
+		path: defaultSandboxDir(project.root),
+		inside: false
+	};
+	const path = resolve(project.root, project.config.sandboxDir);
+	const fromRoot = relative(project.root, path);
+	return {
+		path,
+		inside: !fromRoot.startsWith("..") && !isAbsolute(fromRoot)
+	};
+}
+/** `<tmp>/evalkit/<project name>-<hash of its path>`: stable per checkout. */
+function defaultSandboxDir(root) {
+	const hash = createHash("sha256").update(root).digest("hex").slice(0, 10);
+	const name = basename(root).replace(/[^A-Za-z0-9._-]/g, "-") || "project";
+	return join(tmpdir(), "evalkit", `${name}-${hash}`);
+}
+/** The warning `run` prints when workspaces sit inside the project. */
+const SANDBOX_INSIDE_WARNING = "sandboxDir is inside the project: agents can read its source, its AGENTS.md and skills, its results and other runs’ workspaces from there. Leave sandboxDir unset to keep workspaces in the system temp directory.";
 //#endregion
 //#region ../cli/src/terminal.ts
 /** Human CLI output. Colors and live progress only appear on an interactive terminal. */
@@ -996,25 +1008,23 @@ function cellLabel(name, parameters) {
 	const shown = Object.entries(parameters).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
 	return shown ? `${bold(name)} ${dim(shown)}` : bold(name);
 }
-function passed(result) {
-	return result.status === "completed" && (result.aggregateScoring?.passRate === 1 || Boolean(result.scoring?.passed));
-}
 /** Live log for runs launched from the `evalkit dev` dashboard. */
 const runLog = {
 	started(name, parameters = {}, suiteId) {
 		const suite = suiteId ? dim(` (suite ${suiteId})`) : "";
 		console.log(`${clock()}  ${cyan("▶ run ")} ${cellLabel(name, parameters)}${suite}`);
 	},
-	finished(name, parameters, result, ms) {
-		const ok = passed(result);
-		const trials = result.trials?.length ?? 1;
-		const passedTrials = result.aggregateScoring?.passed ?? (ok ? trials : 0);
-		const status = result.status === "completed" ? ok ? green("✓ pass") : red("✗ fail") : red(`✗ ${result.status}`);
-		console.log(`${clock()}  ${status} ${cellLabel(name, parameters)} ${dim(`${passedTrials}/${trials} trials · ${duration(ms)}`)}`);
-		console.log(`          ${dim(`report ${result.reportLocation}`)}`);
+	/** One line per run; a batch's runs say which one they are. */
+	finished(name, parameters, runs, ms) {
+		for (const result of runs) {
+			const status = result.status === "passed" ? green("✓ pass") : result.status === "failed" ? red("✗ fail") : red(`✗ ${result.status}`);
+			const batch = result.batch ? ` #${result.batch.index + 1}/${result.batch.size}` : "";
+			console.log(`${clock()}  ${status} ${cellLabel(name, parameters)}${dim(`${batch} · ${duration(result.durationMs ?? ms)}`)}`);
+			console.log(`          ${dim(`report ${result.reportLocation}`)}`);
+		}
 	},
-	cell(cell, result, ms) {
-		runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, result, ms);
+	cell(cell, runs, ms) {
+		runLog.finished(cell.eval.name ?? authoringId(cell.eval), cell.parameters, runs, ms);
 	},
 	failed(name, error) {
 		console.log(`${clock()}  ${red("✗ error")} ${bold(name)} ${error instanceof Error ? error.message : String(error)}`);
@@ -1066,72 +1076,88 @@ async function confirm(question) {
 }
 //#endregion
 //#region ../cli/src/run-reporter.ts
-function scenarioOutcome(result) {
+function runOutcome(result) {
 	if (result.status === "cancelled") return "cancelled";
-	return result.status === "completed" && (result.trials ? result.trials.length > 0 && result.trials.every((trial) => trial.status === "completed" && trial.scoring?.passed) : Boolean(result.scoring?.passed)) ? "passed" : "failed";
-}
-function passAtFor(definition, name) {
-	const rule = [...definition.transcript, ...definition.scoring].find((step) => step.kind === "predicate" && step.name === name);
-	return rule?.kind === "predicate" ? rule.passAt : void 0;
+	return result.status === "passed" ? "passed" : "failed";
 }
 function value(score) {
 	return score.value === void 0 ? "" : score.value.toFixed(1);
 }
+/**
+* One failed verdict in words: `history 0.0 (passes at 1): got 2 entries`,
+* or for a choices judge `Correctness C (passes at A or B): wrong product`.
+*/
+function verdictReason(definition, verdict) {
+	const rule = [...definition.transcript, ...definition.scoring].find((step) => step.kind === "judge" && step.name === verdict.name);
+	const answer = rule?.kind === "judge" ? rule.answer : void 0;
+	const mark = verdict.passAt === void 0 ? "" : answer?.kind === "choices" ? ` (passes at ${passMark(verdict.passAt, answer)})` : ` (passes at ${verdict.passAt})`;
+	const why = [criteriaReason(verdict.criteria), firstLine(verdict.explanation)].filter(Boolean).join(" · ");
+	return `${verdict.name} ${verdict.choice ?? value(verdict)}${mark}${why ? `: ${why}` : ""}`;
+}
+/**
+* Which criteria let the score down, so seven checks the app never reached
+* read as one cause (`not reached: saves, history`), not seven failures.
+*/
+function criteriaReason(criteria) {
+	if (!criteria?.length) return "";
+	const failed = criteria.filter((criterion) => "value" in criterion && criterion.value < 1);
+	const unreached = criteria.filter((criterion) => !("value" in criterion));
+	return [failed.length ? `failed ${failed.map((criterion) => {
+		const why = firstLine(criterion.explanation, 60);
+		return why ? `${criterion.name} (${why})` : criterion.name;
+	}).join(", ")}` : "", unreached.length ? `not reached ${unreached.map((criterion) => criterion.name).join(", ")}` : ""].filter(Boolean).join(" · ");
+}
 function scoreReason(definition, score) {
-	if (score.error) return `${score.name} error: ${score.error.message}`;
-	const passAt = passAtFor(definition, score.name);
-	return `${score.name} ${value(score)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+	if (score.error) return `${score.name} error: ${firstLine(score.error.message)}`;
+	return verdictReason(definition, score);
 }
 function checkpointReason(definition, checkpoint) {
-	if (checkpoint.status === "error") return `step ${checkpoint.step} ${checkpoint.name} error: ${checkpoint.error?.message ?? ""}`;
-	const passAt = passAtFor(definition, checkpoint.name);
-	return `step ${checkpoint.step} ${checkpoint.name} ${checkpoint.value === void 0 ? checkpoint.status : value(checkpoint)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+	if (checkpoint.status === "error") return `step ${checkpoint.step} ${checkpoint.name} error: ${firstLine(checkpoint.error?.message)}`;
+	if (checkpoint.value === void 0 && !checkpoint.choice) return `step ${checkpoint.step} ${checkpoint.name} ${checkpoint.status}`;
+	return `step ${checkpoint.step} ${verdictReason(definition, checkpoint)}`;
 }
-/** What went wrong in a scenario, deduplicated across its trials. */
-function reasonsFor(definition, trials) {
+/** What went wrong in a run. */
+function reasonsFor(definition, run) {
 	const reasons = /* @__PURE__ */ new Set();
 	const skipped = /* @__PURE__ */ new Set();
-	for (const trial of trials) {
-		if (trial.status === "failed" && trial.error) reasons.add(`error: ${trial.error.message}`);
-		for (const checkpoint of trial.scoring?.checkpoints ?? []) if (checkpoint.status === "failed" || checkpoint.status === "error") reasons.add(checkpointReason(definition, checkpoint));
-		for (const score of trial.scoring?.results ?? []) if (score.skipped) skipped.add(score.name);
-		else if (!score.passed) reasons.add(scoreReason(definition, score));
-		for (const name of trial.scoring?.skippedScorers ?? []) if (trial.status !== "cancelled") reasons.add(`${name} not run`);
-	}
+	if (run.status === "errored" && run.error) reasons.add(`error: ${firstLine(run.error.message, 160)}`);
+	for (const checkpoint of run.scoring?.checkpoints ?? []) if (checkpoint.status === "failed" || checkpoint.status === "error") reasons.add(checkpointReason(definition, checkpoint));
+	for (const score of run.scoring?.results ?? []) if (score.skipped) skipped.add(score.name);
+	else if (!score.passed) reasons.add(scoreReason(definition, score));
+	for (const name of run.scoring?.skippedScorers ?? []) if (run.status !== "cancelled") reasons.add(`${name} not run`);
 	return {
 		reasons: [...reasons],
 		skipped: [...skipped]
 	};
 }
-function scenarioResult(cell, result, suiteId) {
+function runReport(cell, result, suiteId) {
 	const evalId = authoringId(cell.eval);
-	const trials = result.trials ?? [result];
 	return {
 		label: suiteId ? `${suiteId}/${evalId}` : evalId,
 		evalId,
 		...suiteId ? { suiteId } : {},
 		parameters: cell.parameters,
-		outcome: scenarioOutcome(result),
+		outcome: runOutcome(result),
 		...result.durationMs !== void 0 ? { durationMs: result.durationMs } : {},
 		result,
-		...reasonsFor(cell.eval, trials)
+		...reasonsFor(cell.eval, result)
 	};
 }
 function parameterLabel(parameters) {
 	return Object.entries(parameters).map(([key, item]) => `${key}=${typeof item === "string" ? item : JSON.stringify(item)}`).join(" ");
 }
-/** One printed line for a finished scenario; failing reasons on the same line. */
-function scenarioLine(scenario) {
+/** One printed line for a finished run; failing reasons on the same line. */
+function runLine(scenario) {
 	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : red("✗");
 	const parameters = parameterLabel(scenario.parameters);
-	const trials = scenario.result.trials;
-	const trialCount = trials && trials.length > 1 ? dim(` ${trials.filter((trial) => trial.status === "completed" && trial.scoring?.passed).length}/${trials.length} trials`) : "";
+	const batch = scenario.result.batch;
+	const position = batch ? dim(` #${batch.index + 1}/${batch.size}`) : "";
 	const details = [
 		...scenario.outcome === "failed" ? scenario.reasons : [],
 		...scenario.outcome === "cancelled" ? ["cancelled"] : [],
 		...scenario.skipped.map((name) => `${name} skipped`)
 	];
-	return `  ${mark} ${bold(scenario.label)}${parameters ? ` ${dim(parameters)}` : ""}${trialCount}${details.length ? `   ${dim(details.join(" · "))}` : ""}`;
+	return `  ${mark} ${bold(scenario.label)}${parameters ? ` ${dim(parameters)}` : ""}${position}${details.length ? `   ${dim(details.join(" · "))}` : ""}`;
 }
 /**
 * Prints run progress. On a terminal, failures print as they happen and a live
@@ -1177,29 +1203,29 @@ function createRunReporter(options) {
 				timer.unref?.();
 			}
 		},
-		onEvent(trial, event) {
-			if (event.source === "runner" && event.kind === "trial-started") {
-				running.set(trial.trialId, {
-					label: options.labelFor(trial),
+		onEvent(run, event) {
+			if (event.source === "runner" && event.kind === "run-started") {
+				running.set(run.runId, {
+					label: options.labelFor(run),
 					startedAt: now()
 				});
 				return;
 			}
-			const activity = running.get(trial.trialId);
+			const activity = running.get(run.runId);
 			if (!activity) return;
-			if (event.source === "runner" && (event.kind === "trial-completed" || event.kind === "error")) running.delete(trial.trialId);
+			if (event.source === "runner" && (event.kind === "run-completed" || event.kind === "error")) running.delete(run.runId);
 			else if (event.source === "aut" && event.kind === "tool-call") activity.doing = `tool ${event.name}`;
 			else if (event.source === "runner" && event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
 			else if (event.source === "runner" && event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
 		},
-		onScenario(scenario) {
+		onRun(scenario) {
 			scenarios.push(scenario);
-			for (const trial of scenario.result.trials ?? [scenario.result]) running.delete(trial.trialId);
-			if (!options.interactive || options.verbose || scenario.outcome !== "passed") print(scenarioLine(scenario));
+			running.delete(scenario.result.runId);
+			if (!options.interactive || options.verbose || scenario.outcome !== "passed") print(runLine(scenario));
 		},
 		/** Called on the first Ctrl+C. */
 		cancelling() {
-			print(`\n  ${yellow("◌")} Cancelling: finishing running trials. Press Ctrl+C again to exit now.`);
+			print(`\n  ${yellow("◌")} Cancelling: finishing running runs. Press Ctrl+C again to exit now.`);
 		},
 		finish(durationMs) {
 			if (timer) clearInterval(timer);
@@ -1208,11 +1234,12 @@ function createRunReporter(options) {
 				passed: scenarios.filter((s) => s.outcome === "passed").length,
 				failed: scenarios.filter((s) => s.outcome === "failed").length,
 				cancelled: scenarios.filter((s) => s.outcome === "cancelled").length,
-				scenarios: scenarios.length,
+				runs: scenarios.length,
+				scenarios: new Set(scenarios.map((s) => `${s.label} ${parameterLabel(s.parameters)}`)).size,
 				durationMs
 			};
 		},
-		scenarios: () => scenarios
+		runs: () => scenarios
 	};
 }
 function totalsLine(totals) {
@@ -1221,7 +1248,8 @@ function totalsLine(totals) {
 		totals.failed ? red(`${totals.failed} failed`) : "",
 		totals.cancelled ? yellow(`${totals.cancelled} cancelled`) : ""
 	].filter(Boolean);
-	return `  ${bold(parts.join(" · "))} ${dim(`· ${plural(totals.scenarios, "scenario")} in ${duration(totals.durationMs)}`)}`;
+	const coverage = totals.runs === totals.scenarios ? plural(totals.runs, "run") : `${plural(totals.runs, "run")} of ${plural(totals.scenarios, "scenario")}`;
+	return `  ${bold(parts.join(" · "))} ${dim(`· ${coverage} in ${duration(totals.durationMs)}`)}`;
 }
 //#endregion
 //#region ../cli/src/junit.ts
@@ -1231,8 +1259,8 @@ function escape(text) {
 const seconds = (ms) => ((ms ?? 0) / 1e3).toFixed(3);
 /**
 * A JUnit XML report: one test suite per EvalKit suite (`default` for
-* scenarios without one) and one test case per scenario, per matrix cell
-* when it has parameters.
+* scenarios without one) and one test case per run, named after its scenario
+* and parameters, and its place in a batch (` #2`) when repeated.
 */
 function junitReport(scenarios) {
 	const bySuite = /* @__PURE__ */ new Map();
@@ -1246,7 +1274,8 @@ function junitReport(scenarios) {
 		const time = cases.reduce((sum, c) => sum + (c.durationMs ?? 0), 0);
 		const testcases = cases.map((scenario) => {
 			const parameters = parameterLabel(scenario.parameters);
-			const attributes = `name="${escape(parameters ? `${scenario.evalId} [${parameters}]` : scenario.evalId)}" classname="${escape(suite)}" time="${seconds(scenario.durationMs)}"`;
+			const batch = scenario.result.batch;
+			const attributes = `name="${escape((parameters ? `${scenario.evalId} [${parameters}]` : scenario.evalId) + (batch ? ` #${batch.index + 1}` : ""))}" classname="${escape(suite)}" time="${seconds(scenario.durationMs)}"`;
 			if (scenario.outcome === "failed") return `    <testcase ${attributes}>\n      <failure message="${escape(scenario.reasons.join("; ") || "failed")}">${escape(scenario.reasons.join("\n"))}</failure>\n    </testcase>`;
 			if (scenario.outcome === "cancelled") return `    <testcase ${attributes}>\n      <skipped message="cancelled"/>\n    </testcase>`;
 			if (allSkipped(scenario)) return `    <testcase ${attributes}>\n      <skipped message="${escape(`all scorers skipped: ${scenario.skipped.join(", ")}`)}"/>\n    </testcase>`;
@@ -1256,15 +1285,15 @@ function junitReport(scenarios) {
 	});
 	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed").length}">\n${suites.join("\n")}\n</testsuites>\n`;
 }
-/** True when every score of every trial was skip()ped. */
+/** True when every score was skip()ped. */
 function allSkipped(scenario) {
-	const results = (scenario.result.trials ?? [scenario.result]).flatMap((trial) => trial.scoring?.results ?? []);
+	const results = scenario.result.scoring?.results ?? [];
 	return results.length > 0 && results.every((score) => score.skipped);
 }
 //#endregion
 //#region ../cli/src/version.ts
 /** The CLI version; kept in sync with package.json when releasing. */
-const VERSION = "0.8.0";
+const VERSION = "0.10.0";
 //#endregion
 //#region ../cli/src/run-command.ts
 /** Validates `run` flags into a selection, parameter overrides and limits. */
@@ -1291,28 +1320,27 @@ function normalizeRunOptions(values, positionals) {
 		const index = value.indexOf("=");
 		if (index < 1) throw new Error("--select requires axis=value");
 		const axis = value.slice(0, index);
-		let choice;
+		const raw = value.slice(index + 1);
+		let choices;
 		try {
-			choice = JSON.parse(value.slice(index + 1));
+			choices = [JSON.parse(raw)];
 		} catch {
-			choice = value.slice(index + 1);
+			choices = raw.split(",").filter(Boolean);
 		}
-		(selection[axis] ??= []).push(choice);
+		(selection[axis] ??= []).push(...choices);
 	}
 	for (const value of values.param ?? []) {
 		const index = value.indexOf("=");
 		if (index < 1) throw new Error("--param requires name=JSON");
 		parameters[value.slice(0, index)] = JSON.parse(value.slice(index + 1));
 	}
-	if (values.suite && values.matrix) throw new Error("--suite and --matrix cannot be used together");
-	if ((values.suite || values.matrix) && positionals.length) throw new Error("Use --eval to select evals with --suite or --matrix");
 	return {
 		values,
 		positionals,
 		parameters,
 		selection,
 		concurrency: integer("concurrency"),
-		trials: integer("trials")
+		repeat: integer("repeat")
 	};
 }
 /** Load errors that affect what was asked for, so the run must not pretend it covered them. */
@@ -1333,35 +1361,32 @@ async function runProjectCommand(cwd, parsed) {
 	const project = await loadProject(cwd, values.config);
 	process.chdir(project.root);
 	const registry = project.registry;
-	const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
+	const requestedEvals = [...values.eval ?? [], ...positionals].flatMap((value) => value.split(",").filter(Boolean));
 	let matrix;
 	let suiteId;
 	const quiet = Boolean(values.json || values["dry-run"]);
 	if (!quiet) banner("run", VERSION);
+	const suite = values.suite ? registry.suites.find((item) => item.id === values.suite) : void 0;
 	if (values.suite) {
-		const suite = registry.suites.find((item) => item.id === values.suite);
 		const broken = project.loadErrors.filter((error) => error.suiteId === values.suite);
 		if (!suite && !broken.length) throw new Error(`Unknown suite: ${values.suite}`);
 		suiteId = values.suite;
-		matrix = defineEvalMatrix({
-			id: "default",
-			evals: suite?.evals ?? [],
-			parameters: project.config.matrix?.parameters ?? {},
-			defaults: project.config.matrix?.defaults,
-			cases: project.config.matrix?.cases,
-			exclude: project.config.matrix?.exclude
-		});
-	} else if (values.matrix) {
-		matrix = registry.matrices.find((item) => item.id === values.matrix);
-		if (!matrix) throw new Error(`Unknown matrix: ${values.matrix}`);
-	} else {
-		requestedEvals.push(...positionals.flatMap((value) => value.split(",")));
-		matrix = project.config.matrix ? registry.matrices.at(-1) : defineEvalMatrix({
-			id: "default",
-			evals: registry.evals,
-			parameters: {}
-		});
 	}
+	const configured = values.matrix ? registry.matrices.find((item) => item.id === values.matrix) : project.config.matrix ? registry.matrices.at(-1) : void 0;
+	if (values.matrix && !configured) throw new Error(`Unknown matrix: ${values.matrix}`);
+	if (configured && suite) {
+		const inSuite = new Set(suite.evals.map(authoringId));
+		const both = configured.evals.map(authoringId).filter((id) => inSuite.has(id));
+		if (!both.length) throw new Error(`Matrix ${authoringId(configured)} has no scenarios from suite ${suite.id}`);
+		const outside = requestedEvals.filter((id) => !both.includes(id));
+		if (outside.length) throw new Error(`Not in both matrix ${authoringId(configured)} and suite ${suite.id}: ${outside.join(", ")}`);
+		if (!requestedEvals.length) requestedEvals.push(...both);
+	}
+	matrix = configured ?? defineEvalMatrix({
+		id: "default",
+		evals: values.suite ? suite?.evals ?? [] : registry.evals,
+		parameters: {}
+	});
 	const broken = brokenSelection(project.loadErrors, {
 		...suiteId ? { suiteId } : {},
 		evalIds: requestedEvals,
@@ -1385,14 +1410,18 @@ async function runProjectCommand(cwd, parsed) {
 	else throw new Error(`Unknown matrix axis: ${axis}`);
 	const cells = matrix.count(selection);
 	if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
-	const trials = parsed.trials ?? project.config.execution?.trials;
+	const repeat = parsed.repeat ?? project.config.execution?.repeat;
+	let runs = 0;
+	for (const cell of matrix.cells(selection)) runs += repeat ?? cell.eval.repeat ?? 1;
 	const plan = {
 		matrix: authoringId(matrix),
 		evals: requestedEvals.length ? requestedEvals : matrix.evals.map((definition) => definition.id),
 		cells,
+		runs,
 		parameters: selection.parameters,
 		overrides: selection.overrides,
-		trials: trials ?? "eval policy"
+		repeat: repeat ?? "scenario",
+		scoring: Object.fromEntries(registry.catalog({ root: project.root }).filter((entry) => matrix.evals.some((definition) => definition.id === entry.id)).map((entry) => [entry.id, plannedChecks(entry)]))
 	};
 	if (values["dry-run"]) {
 		console.log(JSON.stringify(plan, null, 2));
@@ -1401,7 +1430,9 @@ async function runProjectCommand(cwd, parsed) {
 	const sweeps = Object.keys(matrix.parameters).length > 0 || Boolean(matrix.cases?.length);
 	if (sweeps && cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
 	const reportDir = resolve(project.root, project.config.reportDir ?? "_evalkit-results");
-	const sandboxDir = resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox");
+	const sandbox = sandboxLocation(project);
+	const sandboxDir = sandbox.path;
+	if (sandbox.inside) console.error(`  ${yellow("!")} ${SANDBOX_INSIDE_WARNING}`);
 	const source = await detectSource(project.root, { outputDirs: [
 		reportDir,
 		sandboxDir,
@@ -1409,14 +1440,15 @@ async function runProjectCommand(cwd, parsed) {
 	] });
 	const suiteOf = new Map(registry.suites.flatMap((suite) => suite.evals.map((evaluation) => [authoringId(evaluation), suite.id])));
 	const suiteFor = (evalId) => suiteId ?? suiteOf.get(evalId);
-	if (!values.json) console.log(`  Running ${plural(cells, "scenario")}${sweeps ? ` from matrix ${plan.matrix}` : ""}${suiteId ? ` from suite ${suiteId}` : ""}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)") + "\n");
+	if (!values.json) console.log(`  Running ${runs === cells ? plural(cells, "scenario") : `${plural(runs, "run")} of ${plural(cells, "scenario")}`}${sweeps ? ` from matrix ${plan.matrix}` : ""}${suiteId ? ` from suite ${suiteId}` : ""}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)") + "\n");
 	const reporter = createRunReporter({
 		interactive: interactive && !values.json,
 		verbose: Boolean(values.verbose),
-		total: cells,
-		labelFor: (trial) => {
-			const suite = suiteFor(trial.evalId);
-			return suite ? `${suite}/${trial.evalId}` : trial.evalId;
+		total: runs,
+		labelFor: (run) => {
+			const suite = suiteFor(run.evalId);
+			const label = suite ? `${suite}/${run.evalId}` : run.evalId;
+			return run.batch ? `${label} #${run.batch.index + 1}/${run.batch.size}` : label;
 		},
 		write: values.json ? () => {} : void 0
 	});
@@ -1433,7 +1465,7 @@ async function runProjectCommand(cwd, parsed) {
 		summary = await Effect.runPromise(runnerLogs(runMatrix(matrix, {
 			selection,
 			concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
-			trials,
+			...repeat ? { repeat } : {},
 			...suiteId ? { suiteId } : {},
 			suiteFor: (cell) => suiteFor(authoringId(cell.eval)),
 			...values.local ? { runtime: "local" } : {},
@@ -1443,20 +1475,24 @@ async function runProjectCommand(cwd, parsed) {
 			workspaceRoot: sandboxDir,
 			keepWorkspaces: project.config.keepSandboxes,
 			artifactExclude: project.config.artifacts?.exclude,
+			attachmentLimits: project.config.artifacts?.attachments,
+			judge: project.config.judge,
 			signal: controller.signal,
-			onEvent: (trial, event) => reporter.onEvent(trial, event),
-			onResult: (cell, result) => {
-				const scenario = scenarioResult(cell, result, suiteFor(authoringId(cell.eval)));
-				if (values.json) console.log(JSON.stringify({
-					cell: {
-						key: cell.key,
-						eval: authoringId(cell.eval),
-						parameters: cell.parameters
-					},
-					outcome: scenario.outcome,
-					result
-				}));
-				reporter.onScenario(scenario);
+			onEvent: (run, event) => reporter.onEvent(run, event),
+			onResult: (cell, results) => {
+				for (const result of results) {
+					const report = runReport(cell, result, suiteFor(authoringId(cell.eval)));
+					if (values.json) console.log(JSON.stringify({
+						cell: {
+							key: cell.key,
+							eval: authoringId(cell.eval),
+							parameters: cell.parameters
+						},
+						outcome: report.outcome,
+						result
+					}));
+					reporter.onRun(report);
+				}
 			}
 		})));
 	} finally {
@@ -1466,7 +1502,7 @@ async function runProjectCommand(cwd, parsed) {
 	if (values.junit) {
 		const file = resolve(cwd, values.junit);
 		await mkdir(dirname(file), { recursive: true });
-		await writeFile(file, junitReport(reporter.scenarios()));
+		await writeFile(file, junitReport(reporter.runs()));
 	}
 	if (values.json) console.log(JSON.stringify({ summary: totals }));
 	else {
@@ -1479,6 +1515,23 @@ async function runProjectCommand(cwd, parsed) {
 	if (controller.signal.aborted) process.exitCode = 130;
 	else if (summary.failed || broken.length) process.exitCode = 1;
 }
+/** A scenario's checkpoints and final scorers, as `--dry-run` lists them. */
+function plannedChecks(entry) {
+	return [...entry.transcript.flatMap((step, index) => step.kind === "checkpoint" ? [{
+		scorer: step.scorer,
+		at: `step ${index}`
+	}] : []), ...entry.scorers.map((scorer) => ({
+		scorer,
+		at: "final"
+	}))].map(({ scorer, at }) => ({
+		name: scorer.name,
+		kind: scorer.kind,
+		at,
+		...scorer.description ? { description: scorer.description } : {},
+		passes: passMark(scorer.passAt, scorer.answer),
+		...scorer.agent ? { agent: scorer.agent.name ?? scorer.agent.id ?? scorer.agent.kind } : {}
+	}));
+}
 //#endregion
 //#region ../cli/src/tidy.ts
 /**
@@ -1498,7 +1551,7 @@ function projectEvalIds(registry) {
 */
 async function tidyReports(options) {
 	const known = projectEvalIds(options.registry);
-	const reports = createDashboardReportReader(options.reportRoot, options.registry);
+	const reports = createDashboardReportReader(options.reportRoot);
 	const result = {
 		archived: [],
 		restored: [],
@@ -1529,19 +1582,19 @@ async function tidyReports(options) {
 	return result;
 }
 /**
-* Remove the trial workspaces `keep` would not have kept: those of passed
-* trials by default. Unfinished trials are never touched, so a run in
-* progress keeps its workspaces; workspaces without a report are left alone.
+* Remove the run workspaces `keep` would not have kept: those of passed runs
+* by default. Unfinished runs are never touched, so a run in progress keeps
+* its workspace; workspaces without a report are left alone.
 */
 async function tidySandboxes(options) {
 	const keep = options.keep ?? "failed";
 	if (keep === "all") return { removed: 0 };
-	const reports = createDashboardReportReader(options.reportRoot, options.registry);
+	const reports = createDashboardReportReader(options.reportRoot);
 	let removed = 0;
-	for (const run of await reports.listRuns()) for (const trial of await reports.listTrials(run.id)) {
-		if (trial.status === "running") continue;
-		if (keep === "failed" && trial.status !== "passed") continue;
-		const workspace = trialWorkspacePath(options.sandboxRoot, trial.id);
+	for (const run of await reports.listRuns()) {
+		if (run.status === "running") continue;
+		if (keep === "failed" && run.status !== "passed") continue;
+		const workspace = runWorkspacePath(options.sandboxRoot, run.id);
 		if (!await stat(workspace).catch(() => void 0)) continue;
 		await rm(workspace, {
 			recursive: true,
@@ -1601,11 +1654,22 @@ function noRunsMessage(selection, head, options) {
 	if (selection.running) return `No finished runs for ${at}: ${plural(selection.running, "run")} still running. Share again when they finish.`;
 	return `No runs for ${at}. Run \`evalkit run\` first${options.all ? "" : ", or pass --all to share runs from other commits"}.`;
 }
-/** Where to look after sharing: the commit, or the repository if several commits were shared. */
-function shareTarget(serverUrl, head, runs) {
-	const commits = new Set(runs.map((run) => run.source.commit));
-	const commit = commits.size === 1 ? `/-/${[...commits][0]}` : "";
-	return new URL(`/${repoKey(head)}${commit}`, serverUrl).href;
+/**
+* A share server API URL. The server may live below a path
+* (`https://example.com/evals`), so paths resolve below `share.url`.
+*/
+function shareApiUrl(serverUrl, path) {
+	return new URL(path, serverUrl.endsWith("/") ? serverUrl : `${serverUrl}/`);
+}
+/**
+* Where to look after sharing: the commit, or the repository if several
+* commits were shared. Both come from a run URL the server returned
+* (`<repo pages>/-/<sha>/runs/<id>`), since only the server knows its layout.
+*/
+function shareTarget(serverUrl, runUrl, runs) {
+	const commitUrl = runUrl?.replace(/\/runs\/[^/]+\/?$/, "");
+	if (!commitUrl || commitUrl === runUrl) return serverUrl;
+	return new Set(runs.map((run) => run.source.commit)).size === 1 ? commitUrl : commitUrl.replace(/\/-\/[0-9a-f]{40}$/, "") || serverUrl;
 }
 /** Build one run's upload bundle: run.json, catalog.json, and its snapshot. */
 async function runBundle(options) {
@@ -1613,7 +1677,10 @@ async function runBundle(options) {
 	const encode = (value) => new TextEncoder().encode(`${JSON.stringify(value)}\n`);
 	files.push({
 		path: "run.json",
-		data: encode(options.run)
+		data: encode({
+			snapshotVersion: 2,
+			...options.run
+		})
 	});
 	const saved = await readFile(resolve(options.reportRoot, options.run.id, "catalog.json")).catch(() => void 0);
 	files.push({
@@ -1658,7 +1725,7 @@ async function resolveShareHeaders(headers) {
 }
 /** Check that the URL is an EvalKit share server and the headers get through. */
 async function checkShareServer(serverUrl, headers) {
-	const response = await fetch(new URL("/api/v1/repos", serverUrl), {
+	const response = await fetch(shareApiUrl(serverUrl, "api/v1/repos"), {
 		headers,
 		redirect: "manual"
 	});
@@ -1681,7 +1748,7 @@ async function uploadBundle(serverUrl, bundle, options = {}) {
 	throw lastError instanceof Error ? lastError : /* @__PURE__ */ new Error(`Upload failed: ${String(lastError)}`);
 }
 async function postBundle(serverUrl, bundle, headers) {
-	const response = await fetch(new URL("/api/v1/uploads", serverUrl), {
+	const response = await fetch(shareApiUrl(serverUrl, "api/v1/uploads"), {
 		method: "POST",
 		headers: {
 			...headers,
@@ -1716,7 +1783,7 @@ async function expectShareServer(response, serverUrl) {
 	if (denied || response.ok && !type.includes("json")) throw new ShareServerError(`${new URL(serverUrl).host} did not answer as an EvalKit share server (${response.status}${type ? `, ${type.split(";")[0]}` : ""}). ` + (denied ? "Check share.headers." : "Check share.url."));
 }
 async function shareRuns(options) {
-	const reports = createDashboardReportReader(options.reportRoot, options.registry);
+	const reports = createDashboardReportReader(options.reportRoot);
 	return options.step("Uploading", async (progress) => {
 		const result = {
 			created: 0,
@@ -1737,6 +1804,7 @@ async function shareRuns(options) {
 				const uploaded = await uploadBundle(options.serverUrl, bundle, { headers: options.headers });
 				result.bytes += bundle.byteLength;
 				result[uploaded.status] += 1;
+				result.url ??= uploaded.url;
 			} catch (error) {
 				if (error instanceof ShareServerError) throw error;
 				result.failed.push({
@@ -1789,7 +1857,7 @@ export default defineConfig({
 	"datasets/greeting/greeting.txt": "Hello\n",
 	"datasets/greeting.dataset.ts": `import { defineDataset } from '@leostera/evalkit';
 
-// Files the agent can read: copied into each trial's workspace under greeting/.
+// Files the agent can read: copied into each run's workspace under greeting/.
 // Paths are relative to the project root.
 export const greeting = defineDataset({
   id: 'greeting',
@@ -1865,18 +1933,17 @@ bun run evalkit build                           # write a read-only, shareable s
 Coding agents can use the EvalKit skills that ship with the package, in
 \`node_modules/@leostera/evalkit/dist/skills/\` (start with \`evalkit/SKILL.md\`).
 
-EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
+EvalKit installs from npm. Reports, run workspaces, and dashboard snapshots stay local
 and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
 scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
 `;
 }
 //#endregion
 //#region ../cli/src/new-project.ts
-const npmDependency = "^0.8.0";
+const npmDependency = "^0.10.0";
 const ignored = [
 	"node_modules/",
 	"_evalkit-results/",
-	"_evalkit-sandbox/",
 	"_evalkit-dashboard/",
 	"_evalkit-share/"
 ];
@@ -1996,24 +2063,59 @@ const FINISHED = /* @__PURE__ */ new Set([
 function unitKey(run) {
 	return `${run.evalId}\0${canonicalParameters(run.parameters ?? {})}`;
 }
-/** The run that represents each unit: the latest finished one, else the latest at all. */
+/**
+* What represents each unit: the latest finished run's batch (its finished
+* runs), or that run alone; else the latest run at all.
+*/
 function latestByUnit(runs) {
-	const byUnit = /* @__PURE__ */ new Map();
-	let ignored = 0;
-	const ordered = [...runs].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+	const latest = /* @__PURE__ */ new Map();
+	const ordered = [...runs].filter((run) => run.status !== "running").sort((a, b) => b.startedAt.localeCompare(a.startedAt));
 	for (const run of ordered) {
-		if (run.status === "running") continue;
 		const key = unitKey(run);
-		const current = byUnit.get(key);
-		if (!current) byUnit.set(key, run);
-		else {
-			ignored += 1;
-			if (!FINISHED.has(current.status) && FINISHED.has(run.status)) byUnit.set(key, run);
-		}
+		const current = latest.get(key);
+		if (!current || !FINISHED.has(current.status) && FINISHED.has(run.status)) latest.set(key, run);
+	}
+	const byUnit = /* @__PURE__ */ new Map();
+	let used = 0;
+	for (const [key, run] of latest) {
+		const members = run.batch ? ordered.filter((other) => other.batch?.id === run.batch.id && FINISHED.has(other.status)) : [run];
+		used += members.length;
+		byUnit.set(key, side(members));
 	}
 	return {
 		byUnit,
-		ignored
+		ignored: ordered.length - used
+	};
+}
+function side(runs) {
+	const scored = runs.filter((run) => run.status !== "errored");
+	const passed = scored.filter((run) => run.status === "passed").length;
+	const values = /* @__PURE__ */ new Map();
+	for (const run of runs) for (const scorer of run.scorers ?? []) {
+		const key = `${scorer.kind}\0${scorer.name}`;
+		const entry = values.get(key) ?? {
+			scorer,
+			sum: 0,
+			n: 0
+		};
+		if (scorer.value !== void 0 && run.status !== "errored") {
+			entry.sum += scorer.value;
+			entry.n += 1;
+		}
+		values.set(key, entry);
+	}
+	const first = runs[0];
+	return {
+		...first,
+		status: !scored.length ? first.status : passed === runs.length ? "passed" : "failed",
+		scorers: [...values.values()].map(({ scorer, sum, n }) => ({
+			name: scorer.name,
+			kind: scorer.kind,
+			...n ? { value: sum / n } : {}
+		})),
+		passed,
+		total: runs.length,
+		runs: [...runs]
 	};
 }
 function scorerSignature(run) {
@@ -2030,8 +2132,9 @@ function classify(base, head) {
 		change: "not-comparable",
 		reason: "different scorers"
 	};
-	if (base.status === "passed" && head.status === "failed") return { change: "regressed" };
-	if (base.status === "failed" && head.status === "passed") return { change: "fixed" };
+	const rate = (side) => side.passed / side.total;
+	if (rate(head) < rate(base)) return { change: "regressed" };
+	if (rate(head) > rate(base)) return { change: "fixed" };
 	return { change: "unchanged" };
 }
 function label(run, suite) {
@@ -2056,8 +2159,8 @@ function suiteScorers(units) {
 				name: scorer.name,
 				kind: scorer.kind
 			});
-			if (scorer.mean === void 0) continue;
-			sides[side].set(key, [...sides[side].get(key) ?? [], scorer.mean]);
+			if (scorer.value === void 0) continue;
+			sides[side].set(key, [...sides[side].get(key) ?? [], scorer.value]);
 		}
 	}
 	for (const unit of units) {
@@ -2111,7 +2214,11 @@ function compareRuns(baseRuns, headRuns) {
 	const changedDatasets = /* @__PURE__ */ new Set();
 	for (const unit of units) {
 		if (!unit.base || !unit.head) continue;
-		const digests = (run) => new Map((run.datasets ?? []).map((d) => [d.id, d.digests.join(",")]));
+		const digests = (side) => {
+			const byId = /* @__PURE__ */ new Map();
+			for (const run of side.runs) for (const dataset of run.datasets ?? []) byId.set(dataset.id, (byId.get(dataset.id) ?? /* @__PURE__ */ new Set()).add(dataset.digest));
+			return new Map([...byId].map(([id, set]) => [id, [...set].sort().join(",")]));
+		};
 		const before = digests(unit.base);
 		const after = digests(unit.head);
 		for (const id of /* @__PURE__ */ new Set([...before.keys(), ...after.keys()])) if (before.get(id) !== after.get(id)) changedDatasets.add(id);
@@ -2120,10 +2227,10 @@ function compareRuns(baseRuns, headRuns) {
 	for (const unit of units) bySuite.set(unit.suite, [...bySuite.get(unit.suite) ?? [], unit]);
 	const suites = [...bySuite].map(([suite, members]) => {
 		const side = (pick) => {
-			const runs = members.flatMap((unit) => unit[pick] ? [unit[pick]] : []);
+			const sides = members.flatMap((unit) => unit[pick] ? [unit[pick]] : []);
 			return {
-				passed: runs.filter((run) => run.status === "passed").length,
-				total: runs.length
+				passed: sides.reduce((sum, entry) => sum + entry.passed, 0),
+				total: sides.reduce((sum, entry) => sum + entry.total, 0)
 			};
 		};
 		return {
@@ -2232,8 +2339,8 @@ function headRuns(runs, head, options) {
 	return runs.filter((run) => run.source !== void 0 && run.source.commit === head.commit && repoKey(run.source) === repoKey(head) && (!run.source.dirty || options.allowDirty) && !run.archived && run.status !== "running");
 }
 /** Base runs from a results directory, e.g. an artifact from the base pipeline. Dirty runs are never a base. */
-async function baseRunsFromDirectory(directory, registry) {
-	return (await createDashboardReportReader(directory, registry).listRuns()).filter((run) => !run.source?.dirty && !run.archived);
+async function baseRunsFromDirectory(directory) {
+	return (await createDashboardReportReader(directory).listRuns()).filter((run) => !run.source?.dirty && !run.archived);
 }
 /** A full SHA, a unique prefix of at least 7 characters, or a ref's latest shared commit. */
 function resolveSharedRef(commits, ref) {
@@ -2260,7 +2367,7 @@ async function getJson(url, serverUrl, headers) {
 /** Base runs shared for `ref` of this repository; `undefined` when the base was never shared. */
 async function baseRunsFromServer(options) {
 	const repo = repoKey(options.head);
-	const api = (path) => new URL(`/api/v1/repos/${repo}/-/${path}`, options.serverUrl);
+	const api = (path) => shareApiUrl(options.serverUrl, `api/v1/repos/${repo}/-/${path}`);
 	const listed = await getJson(api("commits"), options.serverUrl, options.headers);
 	const commits = Array.isArray(listed?.commits) ? listed.commits : [];
 	const commit = resolveSharedRef(commits, options.ref);
@@ -5257,11 +5364,12 @@ function createProgram(actions) {
 		showGlobalOptions: false
 	}).addHelpText("after", `\nRun from your eval project's directory. Docs: https://evalkit.leostera.dev/docs/\n`);
 	program.command("new").description("create an eval project (never overwrites files)").argument("<directory>", "where to create it; use . for the current directory").addHelpText("after", examples(["bunx @leostera/evalkit new ./agent-evals", "evalkit new ."])).action((directory) => actions.create(directory));
-	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run every scenario in this suite").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("--select <axis=value>", "select a value of any matrix axis; repeatable", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --trials <n>", "trials per scenario, overriding config and policy").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--max-tokens <n>", "pass maxTokens to the agent in context.parameters").option("--turn-budget <n>", "pass turnBudget to the agent in context.parameters").option("--chat-timeout-ms <n>", "pass chatTimeoutMs to the agent in context.parameters").option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
+	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run the scenarios in this suite; with -m, only those in both").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("-a, --select <axis=value>", "keep only these values of a matrix axis (a for axis); repeatable, comma-separated", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --repeat <n>", "runs of each scenario, as one batch; overrides config and scenario").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--max-tokens <n>", "pass maxTokens to the agent in context.parameters").option("--turn-budget <n>", "pass turnBudget to the agent in context.parameters").option("--chat-timeout-ms <n>", "pass chatTimeoutMs to the agent in context.parameters").option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
 		"evalkit run",
 		"evalkit run greeting -n 5",
 		"evalkit run -s billing --junit _evalkit-results/junit.xml",
-		"evalkit run -m models --model model-a,model-b --dry-run"
+		"evalkit run -m models --model model-a,model-b --dry-run",
+		"evalkit run -m models -s testing -a model=luna -a toolchain=riot"
 	])).action((evalIds, options) => {
 		setRunnerLogLevel(options.logLevel);
 		return actions.run(runInput(evalIds, options));
@@ -5275,11 +5383,12 @@ function createProgram(actions) {
 		return actions.build(options);
 	});
 	program.command("tidy").description("archive runs of scenarios no longer in the project, and remove sandboxes keepSandboxes would not keep").addOption(configOption()).action((options) => actions.tidy(options));
+	program.command("upgrade").description("convert results saved by an older EvalKit to the current format, in place").addOption(configOption()).option("--dry-run", "show what would change without changing anything").action((options) => actions.upgrade(options));
 	program.command("share").description("upload this commit's results to a share server").addOption(configOption()).option("--server <url>", "share server to use instead of share.url in the config").option("--all", "also upload runs from other commits").option("--allow-dirty", "include runs made with uncommitted changes").option("--no-workspace", "leave out candidate file contents").addHelpText("after", examples(["evalkit share", "evalkit share --server https://evals.example.com"])).action((options) => actions.share({
 		...options,
 		all: Boolean(options.all),
 		allowDirty: Boolean(options.allowDirty)
-	})).command("serve").description("run a share server locally, with no authentication").option("-d, --dir <dir>", "where to store uploads", "_evalkit-share").addOption(portOption(4318)).action((options) => actions.shareServe(options));
+	})).command("serve").description("run a share server locally, with no authentication").option("-d, --dir <dir>", "where to store uploads", "_evalkit-share").addOption(portOption(4318)).option("--base-path <path>", "serve everything below this path, e.g. /evals").option("--repo <key>", "show and accept only this repository, e.g. github.com/acme/agents").action((options) => actions.shareServe(options));
 	program.command("compare").description("compare this commit's results with a base commit's").addOption(configOption()).option("-b, --base <ref>", "base branch, tag or commit, from the share server").option("--base-dir <dir>", "base results from a directory instead").option("--server <url>", "share server to use instead of share.url in the config").option("--fail-on <condition>", "exit 1 on regression, or score-drop=<amount>; repeatable", collect).option("--markdown <file>", "write the comparison as Markdown").option("--json", "print the comparison as JSON").option("--require-base", "exit 1 when the base has no results").option("--allow-dirty", "include runs made with uncommitted changes").addHelpText("after", examples(["evalkit compare -b main", "evalkit compare -b main --fail-on regression --markdown compare.md"])).action((options) => actions.compare({
 		...options,
 		failOn: options.failOn ?? [],
@@ -5303,7 +5412,7 @@ function runInput(evalIds, flags) {
 		"chat-timeout-ms": flags.chatTimeoutMs,
 		"turn-budget": flags.turnBudget,
 		concurrency: flags.concurrency,
-		trials: flags.trials,
+		repeat: flags.repeat,
 		json: flags.json,
 		"dry-run": flags.dryRun,
 		all: flags.all,
@@ -5317,7 +5426,7 @@ function runInput(evalIds, flags) {
 //#region ../cli/src/index.ts
 let projectRoot = process.cwd();
 let reportRoot = resolve(projectRoot, "_evalkit-results");
-let sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
+let sandboxRoot = defaultSandboxDir(projectRoot);
 let project;
 async function loadRegistry() {
 	project ??= await loadProject(projectRoot);
@@ -5343,6 +5452,8 @@ async function runEvals(options) {
 			workspaceRoot: sandboxRoot,
 			keepWorkspaces: project?.config?.keepSandboxes,
 			artifactExclude: project?.config?.artifacts?.exclude,
+			attachmentLimits: project?.config?.artifacts?.attachments,
+			judge: project?.config?.judge,
 			suiteId: options.suiteId,
 			runtime: runtimeFor(evaluation),
 			concurrency: options.concurrency,
@@ -5362,14 +5473,14 @@ async function runProvenance(registry) {
 		catalog: runCatalog(registry, projectRoot)
 	};
 }
-function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
+function registerMatrixRoutes(app, registry, dashboardMatrix, repeat) {
 	let matrixRunning = false;
 	app.get("/v1/matrix", (context) => context.json({ matrix: dashboardMatrix ? {
 		id: dashboardMatrix.id,
 		parameters: dashboardMatrix.parameters,
 		...dashboardMatrix.cases?.length || dashboardMatrix.exclude?.length ? { dimensions: [...Object.keys(dashboardMatrix.cases?.[0] ?? {}), ...Object.keys(dashboardMatrix.parameters)] } : {},
 		...dashboardMatrix.cases?.length || dashboardMatrix.exclude?.length ? { constrained: true } : {},
-		...trials ? { trials } : {}
+		...repeat ? { repeat } : {}
 	} : null }));
 	app.get("/v1/matrix/cells", (context) => {
 		if (!dashboardMatrix) return context.text("Project has no matrix", 404);
@@ -5409,12 +5520,14 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
 			runProvenance(registry).then((provenance) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
 				selection,
 				concurrency: 1,
-				trials,
+				...repeat ? { repeat } : {},
 				...suiteId ? { suiteId } : {},
 				report: localReportStore(reportRoot),
 				workspaceRoot: sandboxRoot,
 				keepWorkspaces: project?.config?.keepSandboxes,
 				artifactExclude: project?.config?.artifacts?.exclude,
+				attachmentLimits: project?.config?.artifacts?.attachments,
+				judge: project?.config?.judge,
 				...provenance,
 				onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
 			})))).catch((error) => runLog.failed(name, error)).finally(() => {
@@ -5465,7 +5578,7 @@ function registerReportRoutes(app, registry, reports) {
 		if (!dataset) return context.text("Unknown dataset", 404);
 		const file = context.req.param("path");
 		try {
-			return new Response(await staticDatasetFile(dataset, projectRoot, file), { headers: { "content-type": fileContentType(file) } });
+			return new Response(await staticDatasetFile(dataset, projectRoot, file), { headers: UNTRUSTED_FILE });
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Unable to read dataset file", 404);
 		}
@@ -5477,44 +5590,44 @@ function registerReportRoutes(app, registry, reports) {
 	app.get("/v1/evals", (context) => context.json({ evals: registry.metadata() }));
 	app.get("/v1/suites", (context) => context.json({ suites: registry.suiteMetadata() }));
 	app.get("/v1/runs", async (context) => context.json({ runs: await reports.listRuns() }));
-	app.get("/v1/runs/:runId/trials", async (context) => {
+	app.get("/v1/runs/:runId", async (context) => {
 		try {
-			return context.json({ trials: await reports.listTrials(context.req.param("runId")) });
+			return context.json(await reports.runDetail(context.req.param("runId")));
 		} catch (error) {
-			return context.text(error instanceof Error ? error.message : "Bad request", 400);
+			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId", async (context) => {
+	app.get("/v1/runs/:runId/workspace", async (context) => {
 		try {
-			return context.json(await reports.trialDetail(context.req.param("runId"), context.req.param("trialId")));
+			return context.json(await reports.listCandidateWorkspace(context.req.param("runId")));
 		} catch (error) {
-			return context.text(error instanceof Error ? error.message : "Not found", 404);
+			return context.text(error instanceof Error ? error.message : "Unable to list workspace", 400);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId/workspace", async (context) => {
+	app.get("/v1/runs/:runId/workspace/:path{.+}", async (context) => {
 		try {
-			return context.json(await reports.listCandidateWorkspace(context.req.param("runId"), context.req.param("trialId")));
+			return await reports.candidateFile(context.req.param("runId"), context.req.param("path"));
 		} catch (error) {
-			return context.text(error instanceof Error ? error.message : "Unable to list workspace", 400);
+			return context.text(error instanceof Error ? error.message : "Unable to read workspace file", 400);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId/workspace/:path{.+}", async (context) => {
+	app.get("/v1/runs/:runId/attachments/:name{.+}", async (context) => {
 		try {
-			return await reports.candidateFile(context.req.param("runId"), context.req.param("trialId"), context.req.param("path"));
+			return await reports.attachmentFile(context.req.param("runId"), context.req.param("name"));
 		} catch (error) {
-			return context.text(error instanceof Error ? error.message : "Unable to read workspace file", 400);
+			return context.text(error instanceof Error ? error.message : "Unable to read attachment", 400);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId/artifacts", async (context) => {
+	app.get("/v1/runs/:runId/artifacts", async (context) => {
 		try {
-			return context.json({ artifacts: await reports.listArtifacts(context.req.param("runId"), context.req.param("trialId")) });
+			return context.json({ artifacts: await reports.listArtifacts(context.req.param("runId")) });
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId/events", async (context) => {
+	app.get("/v1/runs/:runId/events", async (context) => {
 		try {
-			return context.json({ events: await reports.trajectory(context.req.param("runId"), context.req.param("trialId")) });
+			return context.json({ events: await reports.trajectory(context.req.param("runId")) });
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
@@ -5547,7 +5660,7 @@ async function useProject(configPath) {
 	project = await loadProject(projectRoot, configPath);
 	projectRoot = project.root;
 	reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
-	sandboxRoot = resolve(projectRoot, project.config.sandboxDir ?? "_evalkit-sandbox");
+	sandboxRoot = sandboxLocation(project).path;
 }
 /**
 * `--port` is honoured exactly; without it the server takes the first free
@@ -5568,11 +5681,11 @@ async function serveDashboard({ port, open }) {
 	const dashboardRoot = dashboardAssets();
 	const app = new Hono();
 	const dashboardMatrix = project?.config.matrix ? registry.matrices.at(-1) : void 0;
-	registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.trials);
-	registerReportRoutes(app, registry, createDashboardReportReader(reportRoot, registry));
+	registerMatrixRoutes(app, registry, dashboardMatrix, project?.config.execution?.repeat);
+	registerReportRoutes(app, registry, createDashboardReportReader(reportRoot));
 	registerDashboardAssets(app, dashboardRoot);
 	const url = `http://localhost:${(await listen(app.fetch, port, 4317)).port}`;
-	const reader = createDashboardReportReader(reportRoot, registry);
+	const reader = createDashboardReportReader(reportRoot);
 	const runs = (await reader.listRuns()).length;
 	const legacy = reader.legacyRunIds().length;
 	const matrixCells = dashboardMatrix?.count({ parameters: {} });
@@ -5581,7 +5694,7 @@ async function serveDashboard({ port, open }) {
 	printLoadErrors(project?.loadErrors ?? []);
 	row("Local:", cyan(url));
 	row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
-	row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
+	row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden; run \`evalkit upgrade\`` : ""})`)}`);
 	console.log(`\n  ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
 	if (open) openInBrowser(url);
 }
@@ -5639,7 +5752,7 @@ async function loadProjectStep(configPath) {
 */
 async function shouldRunEvals(run) {
 	if (run !== void 0) return run;
-	const runs = (await createDashboardReportReader(reportRoot, project.registry).listRuns()).filter((entry) => !entry.archived).length;
+	const runs = (await createDashboardReportReader(reportRoot).listRuns()).filter((entry) => !entry.archived).length;
 	if (!runs) {
 		console.log(`  ${cyan("◇")} No results yet, so running the evals to have data to build.`);
 		return true;
@@ -5672,7 +5785,7 @@ async function tidyAction(options) {
 	}), (value) => {
 		const keep = project.config.keepSandboxes ?? "failed";
 		if (keep === "all") return "all kept (keepSandboxes: 'all')";
-		return `${plural(value.removed, "workspace")} of ${keep === "none" ? "finished" : "passed"} trials removed`;
+		return `${plural(value.removed, "workspace")} of ${keep === "none" ? "finished" : "passed"} runs removed`;
 	});
 	const byEval = (entries) => [...entries.reduce((groups, entry) => groups.set(entry.evalId, (groups.get(entry.evalId) ?? 0) + 1), /* @__PURE__ */ new Map()).entries()].map(([id, total]) => `    ${bold(id)} ${dim(`(${plural(total, "run")})`)}`).join("\n");
 	console.log("");
@@ -5683,16 +5796,34 @@ async function tidyAction(options) {
 	if (totalArchived) console.log(`\n  ${dim(`Archived runs (${totalArchived}) are hidden in \`evalkit dev\` (filter "Show archived")`)}\n  ${dim("and left out of `evalkit build`. No reports were deleted.")}`);
 	console.log("");
 }
+async function upgradeAction(options) {
+	banner("upgrade", VERSION);
+	const { root, config } = await loadConfig(projectRoot, options.config);
+	reportRoot = resolve(root, config.reportDir ?? "_evalkit-results");
+	const report = await step(options.dryRun ? "Checking results" : "Upgrading results", () => upgradeReports(reportRoot, { dryRun: options.dryRun }), (value) => `${plural(value.upgraded, "run")} from an older EvalKit ${options.dryRun ? "found" : "upgraded"}`);
+	console.log("");
+	if (!report.upgraded) console.log(`  ${cyan("✓")} Every run is already in the current format.`);
+	else {
+		const into = `${plural(report.upgraded, "run")} from older EvalKit versions into ${plural(report.runs, "run")}`;
+		console.log(options.dryRun ? `  ${cyan("◆")} Would upgrade ${into}. Nothing was changed.` : `  ${cyan("✓")} Upgraded ${into}.`);
+		if (report.batches) console.log(`  ${dim(`${plural(report.batches, "run")} had more than one trial; each trial is now a run in a batch.`)}`);
+	}
+	if (report.running.length) console.log(`  ${yellow("◆")} Left ${plural(report.running.length, "run")} still running alone; upgrade again when they finish.`);
+	console.log("");
+}
 async function shareServeAction(options) {
 	const root = resolve(process.cwd(), options.dir);
-	const server = await listen((await createDirectoryShareApp({
+	const url = `http://localhost:${(await listen((await createDirectoryShareApp({
 		dir: root,
-		dashboardRoot: dashboardAssets()
-	})).fetch, options.port, 4318);
+		dashboardRoot: dashboardAssets(),
+		...options.basePath ? { basePath: options.basePath } : {},
+		...options.repo ? { repo: options.repo } : {}
+	})).fetch, options.port, 4318)).port}${normalizeBasePath(options.basePath)}`;
 	banner("share serve", VERSION);
-	console.log(`  ${cyan("➜")}  ${bold("Local:".padEnd(9))}${cyan(`http://localhost:${server.port}`)}`);
+	console.log(`  ${cyan("➜")}  ${bold("Local:".padEnd(9))}${cyan(url)}`);
+	if (options.repo) console.log(`  ${cyan("➜")}  ${bold("Repo:".padEnd(9))}${options.repo}`);
 	console.log(`  ${cyan("➜")}  ${bold("Storage:".padEnd(9))}${relative(process.cwd(), root) || "."}`);
-	console.log(`\n  ${dim(`Upload with share.url = "http://localhost:${server.port}". No authentication: keep it private.`)}\n`);
+	console.log(`\n  ${dim(`Upload with share.url = "${url}". No authentication: keep it private.`)}\n`);
 }
 async function shareAction(options) {
 	const { all, allowDirty } = options;
@@ -5708,7 +5839,7 @@ async function shareAction(options) {
 	] }), (source) => source ? `${repoKey(source)} @ ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : "not a git repository");
 	if (!head) throw new Error("evalkit share needs to know the commit: run it inside a git repository or set EVALKIT_SOURCE_REPO and EVALKIT_SOURCE_COMMIT. `evalkit build` works without one.");
 	const registry = project.registry;
-	const selection = await step("Runs", async () => selectRuns(await createDashboardReportReader(reportRoot, registry).listRuns(), head, {
+	const selection = await step("Runs", async () => selectRuns(await createDashboardReportReader(reportRoot).listRuns(), head, {
 		all,
 		allowDirty
 	}), (value) => selectionSummary(value, { all }));
@@ -5730,7 +5861,7 @@ async function shareAction(options) {
 		step
 	});
 	for (const failure of result.failed) console.log(`  ${yellow("✗")} ${failure.runId.slice(0, 8)} ${dim(failure.error)}`);
-	const target = shareTarget(serverUrl, head, selection.selected);
+	const target = shareTarget(serverUrl, result.url, selection.selected);
 	console.log(`\n  ${bold(result.failed.length ? "Shared with errors" : "Shared")} in ${duration(performance.now() - started)}\n`);
 	console.log(`  ${cyan("➜")}  ${cyan(target)}\n`);
 	if (result.failed.length) process.exitCode = 1;
@@ -5741,17 +5872,16 @@ async function compareAction(options) {
 	const say = options.json ? () => {} : (line = "") => console.log(line);
 	if (!options.json) banner("compare", VERSION);
 	await useProject(options.config);
-	const registry = project.registry;
 	const head = await detectSource(projectRoot, { outputDirs: [
 		reportRoot,
 		sandboxRoot,
 		resolve(projectRoot, "_evalkit-dashboard")
 	] });
 	if (!head) throw new Error("evalkit compare needs to know the commit: run it inside a git repository or set EVALKIT_SOURCE_REPO and EVALKIT_SOURCE_COMMIT.");
-	const headSide = headRuns(await createDashboardReportReader(reportRoot, registry).listRuns(), head, { allowDirty: options.allowDirty });
+	const headSide = headRuns(await createDashboardReportReader(reportRoot).listRuns(), head, { allowDirty: options.allowDirty });
 	let base;
 	if (options.baseDir) {
-		const runs = await baseRunsFromDirectory(resolve(process.cwd(), options.baseDir), registry);
+		const runs = await baseRunsFromDirectory(resolve(process.cwd(), options.baseDir));
 		base = runs.length ? {
 			label: options.baseDir,
 			runs
@@ -5825,6 +5955,7 @@ try {
 		dev: devAction,
 		build: buildAction,
 		tidy: tidyAction,
+		upgrade: upgradeAction,
 		share: shareAction,
 		shareServe: shareServeAction,
 		compare: compareAction