@leostera/evalkit 0.9.0 → 0.11.0

package/dist/cli.mjs

@@ -1,16 +1,18 @@
 #!/bin/sh
 ':' //; exec "${EVALKIT_RUNTIME:-$(command -v bun || command -v node)}" "$0" "$@"
-import { H as canonicalParameters, V as defineEvalMatrix, a as authoringId, c as defineSuite, i as agentRuntimes, m as registerEvals, p as recordError } from "./src-BrfIXfO2.mjs";
-import { a as readRunEvents, c as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, i as ownerIsGone, l as localReportStore, n as upgradeReports, o as readRunManifest, p as globMatcher, r as runMatrix, t as runEval, u as runWorkspacePath } from "./src-CcdLZqFk.mjs";
-import { f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, s as repoKey } from "./index-db-BbVblaHz.mjs";
-import { t as createDirectoryShareApp } from "./src-Bn6fnZnF.mjs";
+import { Et as resolveLimits, Q as canonicalParameters, St as formatLimit, Tt as parseLimits, Z as defineEvalMatrix, _ as recordError, i as agentRuntimes, l as defineSuite, lt as passMark, o as authoringId, v as registerEvals, wt as parseLimitFlag } from "./src-D6Ax2t9b.mjs";
+import { _ as removeRunContainer, a as readRunEvents, c as readRunSubagents, d as localReportStore, f as runWorkspacePath, g as fixtureSet, h as globMatcher, i as ownerIsGone, l as readRunSummary, m as staticDatasetFiles, n as upgradeReports, o as readRunManifest, p as staticDatasetFile, r as runMatrix, t as runEval, u as readRunTranscript } from "./src-Dc3GXacB.mjs";
+import { f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, s as repoKey } from "./index-db-DI7xKMSX.mjs";
+import { t as createDirectoryShareApp } from "./src-CXzbHAKN.mjs";
+import { createHash } from "node:crypto";
 import { Effect, LogLevel, Logger } from "effect";
+import { tmpdir } from "node:os";
 import { cp, lstat, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
-import path, { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
+import path, { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import childProcess, { execFile, spawn } from "node:child_process";
 import fs, { existsSync } from "node:fs";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { Hono } from "hono";
-import childProcess, { execFile, spawn } from "node:child_process";
 import { promisify, stripVTControlCharacters } from "node:util";
 import { createServer } from "node:http";
 import { EventEmitter } from "node:events";
@@ -166,6 +168,19 @@ function shortSource(source) {
 //#endregion
 //#region ../cli/src/dashboard-reports.ts
 const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]|\u001b\][^\u0007]*(?:\u0007|\u001b\\)|\u001b[@-Z\\-_]/g;
+const CONTROL = /[\u0000-\u0008\u000b-\u001f\u007f]/g;
+/**
+* The first non-empty line of a message, without terminal colours, cut to
+* `max` characters: a scorer may explain itself with 300 lines of stderr,
+* and a summary line must stay one line. The run page has the rest.
+*/
+function firstLine(text, max = 100) {
+	const lines = (text ?? "").replace(ANSI, "").split(/\r?\n/).map((line) => line.replace(CONTROL, "").trim()).filter(Boolean);
+	const line = lines[0] ?? "";
+	const more = lines.length > 1 ? " …" : "";
+	return line.length > max ? `${line.slice(0, max - 1)}…` : `${line}${more}`;
+}
 /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
 const ARCHIVE_MARKER = "archived.json";
 function isMissing(error) {
@@ -226,7 +241,7 @@ function createDashboardReportReader(root) {
 		} catch {
 			return false;
 		}
-		return typeof version === "number" && version < 6;
+		return typeof version === "number" && version < 7;
 	}
 	async function readRun(id) {
 		if (await isLegacyRun(id)) {
@@ -256,6 +271,9 @@ function createDashboardReportReader(root) {
 				manifest.aut.version
 			].filter(Boolean).join(" / ") } : {},
 			status: summary ? summary.status : interrupted ? "cancelled" : "running",
+			...manifest.limits ? { limits: manifest.limits } : {},
+			...manifest.isolation ? { isolation: manifest.isolation } : {},
+			...summary?.limit ? { limit: summary.limit } : {},
 			...interrupted ? { interrupted: true } : {},
 			startedAt: manifest.startedAt,
 			...summary?.endedAt ? { completedAt: summary.endedAt } : {},
@@ -266,7 +284,12 @@ function createDashboardReportReader(root) {
 				name: result.name,
 				kind: result.kind,
 				...result.skipped ? { skipped: true } : {},
-				...result.value === void 0 || result.skipped ? {} : { value: result.value }
+				...result.value === void 0 || result.skipped ? {} : { value: result.value },
+				...result.passed === void 0 || result.skipped ? {} : { passed: result.passed },
+				...result.error ? { errored: true } : {},
+				...result.choice ? { choice: result.choice } : {},
+				...result.explanation ? { explanation: firstLine(result.explanation, 200) } : {},
+				...result.judge?.model ? { model: result.judge.model } : {}
 			})) } : {},
 			...scoring?.metrics?.length ? { metrics: scoring.metrics.map((metric) => ({
 				name: metric.name,
@@ -312,9 +335,19 @@ function createDashboardReportReader(root) {
 	}
 	const candidateFile = (runId, path) => artifactFile(runId, "candidate", path);
 	const attachmentFile = (runId, name) => artifactFile(runId, "attachments", name);
-	async function trajectory(runId) {
+	/** The agent's transcript and EvalKit's events, as far as a running run has got. */
+	async function transcript(runId) {
 		runPath(root, runId);
-		return await ifPresent(() => readRunEvents(root, runId)) ?? [];
+		const [recorded, subagents, runEvents] = await Promise.all([
+			ifPresent(() => readRunTranscript(root, runId)),
+			readRunSubagents(root, runId),
+			ifPresent(() => readRunEvents(root, runId))
+		]);
+		return {
+			transcript: recorded ?? null,
+			subagents,
+			runEvents: runEvents ?? []
+		};
 	}
 	return {
 		listRuns,
@@ -325,7 +358,7 @@ function createDashboardReportReader(root) {
 		listCandidateWorkspace,
 		candidateFile,
 		attachmentFile,
-		trajectory
+		transcript
 	};
 }
 /** Both artifact endpoints walk a specific root; never broaden the candidate-file read root. */
@@ -456,13 +489,22 @@ function staticRoutes(ids) {
 		...under("library/scenarios", ids.scenarios),
 		...under("library/agents", ids.agents),
 		...under("library/datasets", ids.datasets),
-		...ids.runs.filter(safe).flatMap((id) => [
-			`runs/${id}`,
-			`runs/${id}/workspace`,
-			`runs/${id}/attachments`
-		])
+		...ids.runs.filter(safe).flatMap((id) => [`runs/${id}`, ...RUN_SECTIONS.map((section) => `runs/${id}/${section}`)])
 	];
 }
+/**
+* A run's sections, each its own page; `attachments` is the logs' URL from
+* before 0.10, kept so old links redirect.
+*/
+const RUN_SECTIONS = [
+	"judging",
+	"metrics",
+	"workspace",
+	"logs",
+	"datasets",
+	"raw",
+	"attachments"
+];
 /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
 function baseFor(route) {
 	const depth = route.split("/").length - 1;
@@ -493,7 +535,7 @@ function directorySink(root) {
 }
 /**
 * One run's snapshot, shared by `evalkit build` (files on disk) and
-* `evalkit share` (an upload bundle): `detail.json`, `events.json`,
+* `evalkit share` (an upload bundle): `detail.json`, `transcript.json`,
 * `artifacts.json` and `workspace.json`, the candidate files under
 * `workspace/`, and attachments under `attachments/` (both left out with
 * `workspaceFiles: false`; they stay listed). Evaluator workspaces are never
@@ -502,7 +544,7 @@ function directorySink(root) {
 async function snapshotRun(reports, runId, sink, options = {}) {
 	const detail = await reports.runDetail(runId);
 	await sink.json("detail.json", detail);
-	await sink.json("events.json", { events: await reports.trajectory(runId) });
+	await sink.json("transcript.json", await reports.transcript(runId));
 	await sink.json("artifacts.json", { artifacts: await reports.listArtifacts(runId) });
 	const workspace = await reports.listCandidateWorkspace(runId);
 	await sink.json("workspace.json", workspace);
@@ -843,6 +885,7 @@ async function loadConfig(cwd, configPath) {
 	const config = file ? (await importProjectModule(file)).default : {};
 	if (!config || typeof config !== "object" || Array.isArray(config)) throw new Error("Config must default-export defineConfig({...})");
 	if (config.execution && "trials" in config.execution) throw new Error("execution.trials was renamed to execution.repeat in EvalKit 0.9");
+	if (config.judge !== void 0 && typeof config.judge?.start !== "function") throw new Error("judge in the config must be an agent made with defineAgent()");
 	for (const [key, value] of Object.entries(config.execution ?? {})) if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) throw new Error(`execution.${key} must be a positive integer`);
 	return {
 		root,
@@ -880,6 +923,17 @@ async function loadProject(cwd, configPath) {
 			matrices: [...registry.matrices, matrix]
 		};
 	}
+	const judge = config.judge;
+	if (judge) {
+		const catalog = registry.catalog;
+		registry = {
+			...registry,
+			catalog: (options) => catalog({
+				judge,
+				...options
+			})
+		};
+	}
 	return {
 		root,
 		config,
@@ -887,6 +941,34 @@ async function loadProject(cwd, configPath) {
 		loadErrors
 	};
 }
+/**
+* Where a project's run workspaces live. By default outside the project, in
+* the system temp directory (one directory per project, so `tidy` and `dev`
+* find the ones it kept): an agent that looks around its workspace finds no
+* project source, no `AGENTS.md` or skills above it, and no results.
+* `sandboxDir` overrides it; `inside` says when that puts them back in the
+* project, which `run` warns about.
+*/
+function sandboxLocation(project) {
+	if (project.config.sandboxDir === void 0) return {
+		path: defaultSandboxDir(project.root),
+		inside: false
+	};
+	const path = resolve(project.root, project.config.sandboxDir);
+	const fromRoot = relative(project.root, path);
+	return {
+		path,
+		inside: !fromRoot.startsWith("..") && !isAbsolute(fromRoot)
+	};
+}
+/** `<tmp>/evalkit/<project name>-<hash of its path>`: stable per checkout. */
+function defaultSandboxDir(root) {
+	const hash = createHash("sha256").update(root).digest("hex").slice(0, 10);
+	const name = basename(root).replace(/[^A-Za-z0-9._-]/g, "-") || "project";
+	return join(tmpdir(), "evalkit", `${name}-${hash}`);
+}
+/** The warning `run` prints when workspaces sit inside the project. */
+const SANDBOX_INSIDE_WARNING = "sandboxDir is inside the project: agents can read its source, its AGENTS.md and skills, its results and other runs’ workspaces from there. Leave sandboxDir unset to keep workspaces in the system temp directory.";
 //#endregion
 //#region ../cli/src/terminal.ts
 /** Human CLI output. Colors and live progress only appear on an interactive terminal. */
@@ -910,8 +992,8 @@ function bytes(size) {
 	if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
 	return `${(size / 1024 ** 2).toFixed(1)} MB`;
 }
-function plural(count, noun) {
-	return `${count} ${noun}${count === 1 ? "" : "s"}`;
+function plural(count, noun, nouns = `${noun}s`) {
+	return `${count} ${count === 1 ? noun : nouns}`;
 }
 /** Times one build step, showing a live line while it runs. */
 async function step(label, work, describe) {
@@ -1009,30 +1091,50 @@ async function confirm(question) {
 //#region ../cli/src/run-reporter.ts
 function runOutcome(result) {
 	if (result.status === "cancelled") return "cancelled";
+	if (result.status === "limited") return "limited";
 	return result.status === "passed" ? "passed" : "failed";
 }
-function passAtFor(definition, name) {
-	const rule = [...definition.transcript, ...definition.scoring].find((step) => step.kind === "predicate" && step.name === name);
-	return rule?.kind === "predicate" ? rule.passAt : void 0;
-}
 function value(score) {
 	return score.value === void 0 ? "" : score.value.toFixed(1);
 }
+/**
+* One failed verdict in words: `history 0.0 (passes at 1): got 2 entries`,
+* or for a choices judge `Correctness C (passes at A or B): wrong product`.
+*/
+function verdictReason(definition, verdict) {
+	const rule = [...definition.transcript, ...definition.scoring].find((step) => step.kind === "judge" && step.name === verdict.name);
+	const answer = rule?.kind === "judge" ? rule.answer : void 0;
+	const mark = verdict.passAt === void 0 ? "" : answer?.kind === "choices" ? ` (passes at ${passMark(verdict.passAt, answer)})` : ` (passes at ${verdict.passAt})`;
+	const why = [criteriaReason(verdict.criteria), firstLine(verdict.explanation)].filter(Boolean).join(" · ");
+	return `${verdict.name} ${verdict.choice ?? value(verdict)}${mark}${why ? `: ${why}` : ""}`;
+}
+/**
+* Which criteria let the score down, so seven checks the app never reached
+* read as one cause (`not reached: saves, history`), not seven failures.
+*/
+function criteriaReason(criteria) {
+	if (!criteria?.length) return "";
+	const failed = criteria.filter((criterion) => "value" in criterion && criterion.value < 1);
+	const unreached = criteria.filter((criterion) => !("value" in criterion));
+	return [failed.length ? `failed ${failed.map((criterion) => {
+		const why = firstLine(criterion.explanation, 60);
+		return why ? `${criterion.name} (${why})` : criterion.name;
+	}).join(", ")}` : "", unreached.length ? `not reached ${unreached.map((criterion) => criterion.name).join(", ")}` : ""].filter(Boolean).join(" · ");
+}
 function scoreReason(definition, score) {
-	if (score.error) return `${score.name} error: ${score.error.message}`;
-	const passAt = passAtFor(definition, score.name);
-	return `${score.name} ${value(score)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+	if (score.error) return `${score.name} error: ${firstLine(score.error.message)}`;
+	return verdictReason(definition, score);
 }
 function checkpointReason(definition, checkpoint) {
-	if (checkpoint.status === "error") return `step ${checkpoint.step} ${checkpoint.name} error: ${checkpoint.error?.message ?? ""}`;
-	const passAt = passAtFor(definition, checkpoint.name);
-	return `step ${checkpoint.step} ${checkpoint.name} ${checkpoint.value === void 0 ? checkpoint.status : value(checkpoint)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+	if (checkpoint.status === "error") return `step ${checkpoint.step} ${checkpoint.name} error: ${firstLine(checkpoint.error?.message)}`;
+	if (checkpoint.value === void 0 && !checkpoint.choice) return `step ${checkpoint.step} ${checkpoint.name} ${checkpoint.status}`;
+	return `step ${checkpoint.step} ${verdictReason(definition, checkpoint)}`;
 }
 /** What went wrong in a run. */
 function reasonsFor(definition, run) {
 	const reasons = /* @__PURE__ */ new Set();
 	const skipped = /* @__PURE__ */ new Set();
-	if (run.status === "errored" && run.error) reasons.add(`error: ${run.error.message}`);
+	if (run.status === "errored" && run.error) reasons.add(`error: ${firstLine(run.error.message, 160)}`);
 	for (const checkpoint of run.scoring?.checkpoints ?? []) if (checkpoint.status === "failed" || checkpoint.status === "error") reasons.add(checkpointReason(definition, checkpoint));
 	for (const score of run.scoring?.results ?? []) if (score.skipped) skipped.add(score.name);
 	else if (!score.passed) reasons.add(scoreReason(definition, score));
@@ -1060,17 +1162,26 @@ function parameterLabel(parameters) {
 }
 /** One printed line for a finished run; failing reasons on the same line. */
 function runLine(scenario) {
-	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : red("✗");
+	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : scenario.outcome === "limited" ? yellow("✗") : red("✗");
 	const parameters = parameterLabel(scenario.parameters);
 	const batch = scenario.result.batch;
 	const position = batch ? dim(` #${batch.index + 1}/${batch.size}`) : "";
 	const details = [
-		...scenario.outcome === "failed" ? scenario.reasons : [],
+		...scenario.outcome === "limited" ? [limitReason(scenario.result)] : [],
+		...scenario.outcome === "failed" || scenario.outcome === "limited" ? scenario.reasons : [],
 		...scenario.outcome === "cancelled" ? ["cancelled"] : [],
 		...scenario.skipped.map((name) => `${name} skipped`)
 	];
 	return `  ${mark} ${bold(scenario.label)}${parameters ? ` ${dim(parameters)}` : ""}${position}${details.length ? `   ${dim(details.join(" · "))}` : ""}`;
 }
+/** `limited: 41 of 40 steps · score 0.40 (graded what it left)`. */
+function limitReason(result) {
+	const limit = result.limit;
+	if (!limit) return "limited";
+	const used = limit.limit === "time" ? `${formatLimit("time", Math.round(limit.used / 1e3) * 1e3)} of ${formatLimit("time", limit.value)}` : `${limit.used.toLocaleString("en-US")} of ${formatLimit(limit.limit, limit.value)}`;
+	const overall = result.scoring?.overall;
+	return `limited: ${used}${overall === void 0 || !result.scoring?.results.length ? "" : ` · score ${overall.toFixed(2)} (graded what it left)`}`;
+}
 /**
 * Prints run progress. On a terminal, failures print as they happen and a live
 * line shows counts and what is running. Elsewhere (CI), every finished
@@ -1115,8 +1226,15 @@ function createRunReporter(options) {
 				timer.unref?.();
 			}
 		},
-		onEvent(run, event) {
-			if (event.source === "runner" && event.kind === "run-started") {
+		onUpdate(run, update) {
+			if (update.kind === "record") {
+				const activity = running.get(run.runId);
+				const call = update.record.type === "message" && update.record.role === "assistant" ? update.record.content.filter((block) => block.type === "tool_call").at(-1) : void 0;
+				if (activity && call?.type === "tool_call") activity.doing = `tool ${call.name}`;
+				return;
+			}
+			const event = update.event;
+			if (event.kind === "run-started") {
 				running.set(run.runId, {
 					label: options.labelFor(run),
 					startedAt: now()
@@ -1125,10 +1243,10 @@ function createRunReporter(options) {
 			}
 			const activity = running.get(run.runId);
 			if (!activity) return;
-			if (event.source === "runner" && (event.kind === "run-completed" || event.kind === "error")) running.delete(run.runId);
-			else if (event.source === "aut" && event.kind === "tool-call") activity.doing = `tool ${event.name}`;
-			else if (event.source === "runner" && event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
-			else if (event.source === "runner" && event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
+			if (event.kind === "run-completed" || event.kind === "error") running.delete(run.runId);
+			else if (event.kind === "limit-reached") activity.doing = `stopping at its ${event.limit} limit`;
+			else if (event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
+			else if (event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
 		},
 		onRun(scenario) {
 			scenarios.push(scenario);
@@ -1145,6 +1263,7 @@ function createRunReporter(options) {
 			return {
 				passed: scenarios.filter((s) => s.outcome === "passed").length,
 				failed: scenarios.filter((s) => s.outcome === "failed").length,
+				limited: scenarios.filter((s) => s.outcome === "limited").length,
 				cancelled: scenarios.filter((s) => s.outcome === "cancelled").length,
 				runs: scenarios.length,
 				scenarios: new Set(scenarios.map((s) => `${s.label} ${parameterLabel(s.parameters)}`)).size,
@@ -1158,6 +1277,7 @@ function totalsLine(totals) {
 	const parts = [
 		green(`${totals.passed} passed`),
 		totals.failed ? red(`${totals.failed} failed`) : "",
+		totals.limited ? yellow(`${totals.limited} limited`) : "",
 		totals.cancelled ? yellow(`${totals.cancelled} cancelled`) : ""
 	].filter(Boolean);
 	const coverage = totals.runs === totals.scenarios ? plural(totals.runs, "run") : `${plural(totals.runs, "run")} of ${plural(totals.scenarios, "scenario")}`;
@@ -1181,13 +1301,17 @@ function junitReport(scenarios) {
 		bySuite.set(suite, [...bySuite.get(suite) ?? [], scenario]);
 	}
 	const suites = [...bySuite].map(([suite, cases]) => {
-		const failures = cases.filter((c) => c.outcome === "failed").length;
+		const failures = cases.filter((c) => c.outcome === "failed" || c.outcome === "limited").length;
 		const skipped = cases.filter((c) => c.outcome === "cancelled" || c.outcome === "passed" && c.skipped.length > 0 && allSkipped(c)).length;
 		const time = cases.reduce((sum, c) => sum + (c.durationMs ?? 0), 0);
 		const testcases = cases.map((scenario) => {
 			const parameters = parameterLabel(scenario.parameters);
 			const batch = scenario.result.batch;
 			const attributes = `name="${escape((parameters ? `${scenario.evalId} [${parameters}]` : scenario.evalId) + (batch ? ` #${batch.index + 1}` : ""))}" classname="${escape(suite)}" time="${seconds(scenario.durationMs)}"`;
+			if (scenario.outcome === "limited") {
+				const reasons = [limitReason(scenario.result), ...scenario.reasons];
+				return `    <testcase ${attributes}>\n      <failure message="${escape(reasons.join("; "))}" type="limited">${escape(reasons.join("\n"))}</failure>\n    </testcase>`;
+			}
 			if (scenario.outcome === "failed") return `    <testcase ${attributes}>\n      <failure message="${escape(scenario.reasons.join("; ") || "failed")}">${escape(scenario.reasons.join("\n"))}</failure>\n    </testcase>`;
 			if (scenario.outcome === "cancelled") return `    <testcase ${attributes}>\n      <skipped message="cancelled"/>\n    </testcase>`;
 			if (allSkipped(scenario)) return `    <testcase ${attributes}>\n      <skipped message="${escape(`all scorers skipped: ${scenario.skipped.join(", ")}`)}"/>\n    </testcase>`;
@@ -1195,7 +1319,7 @@ function junitReport(scenarios) {
 		});
 		return `  <testsuite name="${escape(suite)}" tests="${cases.length}" failures="${failures}" errors="0" skipped="${skipped}" time="${seconds(time)}">\n${testcases.join("\n")}\n  </testsuite>`;
 	});
-	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed").length}">\n${suites.join("\n")}\n</testsuites>\n`;
+	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed" || c.outcome === "limited").length}">\n${suites.join("\n")}\n</testsuites>\n`;
 }
 /** True when every score was skip()ped. */
 function allSkipped(scenario) {
@@ -1203,9 +1327,71 @@ function allSkipped(scenario) {
 	return results.length > 0 && results.every((score) => score.skipped);
 }
 //#endregion
+//#region ../cli/src/run-options.ts
+/**
+* What every run of the project shares, from its config, wherever it is
+* launched: the CLI's `run` and the dashboard's buttons.
+*/
+function projectRunOptions(root, config) {
+	return {
+		root,
+		...config?.isolation ? { isolation: config.isolation } : {},
+		limits: { base: parseLimits(config?.limits, "The config") },
+		...config?.keepSandboxes ? { keepWorkspaces: config.keepSandboxes } : {},
+		...config?.artifacts?.exclude ? { artifactExclude: config.artifacts.exclude } : {},
+		...config?.artifacts?.attachments ? { attachmentLimits: config.artifacts.attachments } : {},
+		...config?.judge ? { judge: config.judge } : {}
+	};
+}
+/**
+* The project's fixtures for one invocation, or undefined when it has
+* none. The caller stops them when its runs are done.
+*/
+function projectFixtures(root, config) {
+	const fixtures = config?.fixtures;
+	if (!fixtures || !Object.keys(fixtures).length) return void 0;
+	return fixtureSet(fixtures, { root });
+}
+/** Runs `work` with the project's fixtures, stopping them afterwards. */
+async function withFixtures(root, config, work) {
+	const fixtures = projectFixtures(root, config);
+	try {
+		return await work(fixtures);
+	} finally {
+		await fixtures?.stop();
+	}
+}
+//#endregion
+//#region ../cli/src/package-manager.ts
+const LOCKFILES = [
+	["bun.lock", "bun"],
+	["bun.lockb", "bun"],
+	["pnpm-lock.yaml", "pnpm"],
+	["yarn.lock", "yarn"],
+	["package-lock.json", "npm"]
+];
+/**
+* The package manager a project uses: the one that started this process
+* (`npm_config_user_agent`), else the nearest lockfile at or above `root`
+* (a workspace keeps it at its root), else bun. Suggesting another manager's
+* command can make someone run its install and rewrite `node_modules`.
+*/
+function detectPackageManager(root, userAgent = process.env.npm_config_user_agent) {
+	const started = userAgent?.split("/")[0];
+	if (started === "bun" || started === "pnpm" || started === "yarn" || started === "npm") return started;
+	for (let dir = root;; dir = dirname(dir)) {
+		for (const [file, manager] of LOCKFILES) if (existsSync(join(dir, file))) return manager;
+		if (dirname(dir) === dir) return "bun";
+	}
+}
+/** `pnpm run evalkit dev` (or `npm run evalkit -- dev`: npm needs the `--` for flags). */
+function evalkitCommand(manager, args) {
+	return manager === "npm" ? `npm run evalkit -- ${args}` : `${manager} run evalkit ${args}`;
+}
+//#endregion
 //#region ../cli/src/version.ts
 /** The CLI version; kept in sync with package.json when releasing. */
-const VERSION = "0.9.0";
+const VERSION = "0.11.0";
 //#endregion
 //#region ../cli/src/run-command.ts
 /** Validates `run` flags into a selection, parameter overrides and limits. */
@@ -1218,14 +1404,7 @@ function normalizeRunOptions(values, positionals) {
 		if (!Number.isSafeInteger(number) || number < 1) throw new Error(`--${name} must be a positive integer`);
 		return number;
 	}
-	for (const [flag, key] of [
-		["max-tokens", "maxTokens"],
-		["chat-timeout-ms", "chatTimeoutMs"],
-		["turn-budget", "turnBudget"]
-	]) {
-		const value = integer(flag);
-		if (value !== void 0) parameters[key] = value;
-	}
+	const limits = resolveLimits(...(values.limit ?? []).flatMap((value) => value.split(",")).map(parseLimitFlag));
 	const selection = {};
 	for (const key of ["model", "mode"]) if (values[key]?.length) selection[key] = values[key].flatMap((value) => value.split(","));
 	for (const value of values.select ?? []) {
@@ -1251,6 +1430,7 @@ function normalizeRunOptions(values, positionals) {
 		positionals,
 		parameters,
 		selection,
+		limits,
 		concurrency: integer("concurrency"),
 		repeat: integer("repeat")
 	};
@@ -1323,6 +1503,7 @@ async function runProjectCommand(cwd, parsed) {
 	const cells = matrix.count(selection);
 	if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
 	const repeat = parsed.repeat ?? project.config.execution?.repeat;
+	const configLimits = parseLimits(project.config.limits, "The config");
 	let runs = 0;
 	for (const cell of matrix.cells(selection)) runs += repeat ?? cell.eval.repeat ?? 1;
 	const plan = {
@@ -1332,7 +1513,14 @@ async function runProjectCommand(cwd, parsed) {
 		runs,
 		parameters: selection.parameters,
 		overrides: selection.overrides,
-		repeat: repeat ?? "scenario"
+		repeat: repeat ?? "scenario",
+		limits: {
+			config: configLimits,
+			cli: parsed.limits
+		},
+		isolation: project.config.isolation ?? { kind: "none" },
+		fixtures: Object.keys(project.config.fixtures ?? {}),
+		scoring: Object.fromEntries(registry.catalog({ root: project.root }).filter((entry) => matrix.evals.some((definition) => definition.id === entry.id)).map((entry) => [entry.id, plannedChecks(entry)]))
 	};
 	if (values["dry-run"]) {
 		console.log(JSON.stringify(plan, null, 2));
@@ -1341,7 +1529,9 @@ async function runProjectCommand(cwd, parsed) {
 	const sweeps = Object.keys(matrix.parameters).length > 0 || Boolean(matrix.cases?.length);
 	if (sweeps && cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
 	const reportDir = resolve(project.root, project.config.reportDir ?? "_evalkit-results");
-	const sandboxDir = resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox");
+	const sandbox = sandboxLocation(project);
+	const sandboxDir = sandbox.path;
+	if (sandbox.inside) console.error(`  ${yellow("!")} ${SANDBOX_INSIDE_WARNING}`);
 	const source = await detectSource(project.root, { outputDirs: [
 		reportDir,
 		sandboxDir,
@@ -1371,7 +1561,9 @@ async function runProjectCommand(cwd, parsed) {
 	reporter.start();
 	let summary;
 	try {
-		summary = await Effect.runPromise(runnerLogs(runMatrix(matrix, {
+		summary = await withFixtures(project.root, project.config, (fixtures) => Effect.runPromise(runnerLogs(runMatrix(matrix, {
+			...projectRunOptions(project.root, project.config),
+			...fixtures ? { fixtures } : {},
 			selection,
 			concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
 			...repeat ? { repeat } : {},
@@ -1382,11 +1574,12 @@ async function runProjectCommand(cwd, parsed) {
 			catalog: runCatalog(registry, project.root),
 			report: localReportStore(reportDir),
 			workspaceRoot: sandboxDir,
-			keepWorkspaces: project.config.keepSandboxes,
-			artifactExclude: project.config.artifacts?.exclude,
-			attachmentLimits: project.config.artifacts?.attachments,
 			signal: controller.signal,
-			onEvent: (run, event) => reporter.onEvent(run, event),
+			limits: {
+				base: configLimits,
+				override: parsed.limits
+			},
+			onUpdate: (run, update) => reporter.onUpdate(run, update),
 			onResult: (cell, results) => {
 				for (const result of results) {
 					const report = runReport(cell, result, suiteFor(authoringId(cell.eval)));
@@ -1402,7 +1595,7 @@ async function runProjectCommand(cwd, parsed) {
 					reporter.onRun(report);
 				}
 			}
-		})));
+		}))));
 	} finally {
 		process.off("SIGINT", onInterrupt);
 	}
@@ -1416,12 +1609,31 @@ async function runProjectCommand(cwd, parsed) {
 	else {
 		console.log(`\n${totalsLine(totals)}`);
 		if (values.junit) console.log(`  ${dim(`JUnit report: ${values.junit}`)}`);
-		console.log(`  ${cyan("➜")} bun run evalkit dev        ${dim("to inspect")}`);
-		if (project.config.share) console.log(`  ${cyan("➜")} bun run evalkit share      ${dim("to publish")}`);
+		const manager = detectPackageManager(project.root);
+		const hint = (args, why) => console.log(`  ${cyan("➜")} ${evalkitCommand(manager, args).padEnd(24)} ${dim(why)}`);
+		hint("dev", "to inspect");
+		if (project.config.share) hint("share", "to publish");
 		console.log("");
 	}
 	if (controller.signal.aborted) process.exitCode = 130;
-	else if (summary.failed || broken.length) process.exitCode = 1;
+	else if (summary.failed || summary.limited || broken.length) process.exitCode = 1;
+}
+/** A scenario's checkpoints and final scorers, as `--dry-run` lists them. */
+function plannedChecks(entry) {
+	return [...entry.transcript.flatMap((step, index) => step.kind === "checkpoint" ? [{
+		scorer: step.scorer,
+		at: `step ${index}`
+	}] : []), ...entry.scorers.map((scorer) => ({
+		scorer,
+		at: "final"
+	}))].map(({ scorer, at }) => ({
+		name: scorer.name,
+		kind: scorer.kind,
+		at,
+		...scorer.description ? { description: scorer.description } : {},
+		passes: passMark(scorer.passAt, scorer.answer),
+		...scorer.agent ? { agent: scorer.agent.name ?? scorer.agent.id ?? scorer.agent.kind } : {}
+	}));
 }
 //#endregion
 //#region ../cli/src/tidy.ts
@@ -1473,8 +1685,8 @@ async function tidyReports(options) {
 	return result;
 }
 /**
-* Remove the run workspaces `keep` would not have kept: those of passed runs
-* by default. Unfinished runs are never touched, so a run in progress keeps
+* Remove the run workspaces `keep` would not have kept, and their kept
+* containers: those of passed runs by default. Unfinished runs are never touched, so a run in progress keeps
 * its workspace; workspaces without a report are left alone.
 */
 async function tidySandboxes(options) {
@@ -1491,6 +1703,7 @@ async function tidySandboxes(options) {
 			recursive: true,
 			force: true
 		});
+		if (run.isolation?.kind === "container") await removeRunContainer(run.id, options.runtime);
 		removed += 1;
 	}
 	return { removed };
@@ -1569,7 +1782,7 @@ async function runBundle(options) {
 	files.push({
 		path: "run.json",
 		data: encode({
-			snapshotVersion: 2,
+			snapshotVersion: 3,
 			...options.run
 		})
 	});
@@ -1764,7 +1977,7 @@ export const greetingAgent = defineAgent({
   identity: { id: 'greeting-agent', kind: 'in-process' },
   async start({ context }) {
     return {
-      // Return the reply; EvalKit records it. Call onEvent instead to stream events.
+      // Return the reply; EvalKit records it. Use the transcript writer in start() to record tool calls and usage.
       async send(message: string) {
         const file = join(context.workspace.root, 'greeting', 'greeting.txt');
         const prefix = (await readFile(file, 'utf8')).trim();
@@ -1831,11 +2044,10 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
 }
 //#endregion
 //#region ../cli/src/new-project.ts
-const npmDependency = "^0.9.0";
+const npmDependency = "^0.11.0";
 const ignored = [
 	"node_modules/",
 	"_evalkit-results/",
-	"_evalkit-sandbox/",
 	"_evalkit-dashboard/",
 	"_evalkit-share/"
 ];
@@ -1947,9 +2159,11 @@ async function writeStarterReadme(root, name, inPlace) {
 }
 //#endregion
 //#region ../cli/src/compare.ts
+/** Runs that ended on their own; a limited run is a finished failure. */
 const FINISHED = /* @__PURE__ */ new Set([
 	"passed",
 	"failed",
+	"limited",
 	"errored"
 ]);
 function unitKey(run) {
@@ -5256,12 +5470,13 @@ function createProgram(actions) {
 		showGlobalOptions: false
 	}).addHelpText("after", `\nRun from your eval project's directory. Docs: https://evalkit.leostera.dev/docs/\n`);
 	program.command("new").description("create an eval project (never overwrites files)").argument("<directory>", "where to create it; use . for the current directory").addHelpText("after", examples(["bunx @leostera/evalkit new ./agent-evals", "evalkit new ."])).action((directory) => actions.create(directory));
-	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run the scenarios in this suite; with -m, only those in both").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("-a, --select <axis=value>", "keep only these values of a matrix axis (a for axis); repeatable, comma-separated", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --repeat <n>", "runs of each scenario, as one batch; overrides config and scenario").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--max-tokens <n>", "pass maxTokens to the agent in context.parameters").option("--turn-budget <n>", "pass turnBudget to the agent in context.parameters").option("--chat-timeout-ms <n>", "pass chatTimeoutMs to the agent in context.parameters").option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
+	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run the scenarios in this suite; with -m, only those in both").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("-a, --select <axis=value>", "keep only these values of a matrix axis (a for axis); repeatable, comma-separated; one value per axis runs one cell", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --repeat <n>", "runs of each scenario, as one batch; overrides config and scenario").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--limit <kind=value>", "stop each run past this limit: steps=80, tokens=2000000, time=20m; repeatable; overrides config, scenario and matrix", collect).option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
 		"evalkit run",
 		"evalkit run greeting -n 5",
 		"evalkit run -s billing --junit _evalkit-results/junit.xml",
 		"evalkit run -m models --model model-a,model-b --dry-run",
-		"evalkit run -m models -s testing -a model=luna -a toolchain=riot"
+		"evalkit run -m models -s testing -a model=luna -a toolchain=riot",
+		"evalkit run todo-app -a model=luna -a skills=none -a docs=search"
 	])).action((evalIds, options) => {
 		setRunnerLogLevel(options.logLevel);
 		return actions.run(runInput(evalIds, options));
@@ -5300,9 +5515,7 @@ function runInput(evalIds, flags) {
 		mode: flags.mode,
 		select: flags.select,
 		param: flags.param,
-		"max-tokens": flags.maxTokens,
-		"chat-timeout-ms": flags.chatTimeoutMs,
-		"turn-budget": flags.turnBudget,
+		limit: flags.limit,
 		concurrency: flags.concurrency,
 		repeat: flags.repeat,
 		json: flags.json,
@@ -5318,7 +5531,7 @@ function runInput(evalIds, flags) {
 //#region ../cli/src/index.ts
 let projectRoot = process.cwd();
 let reportRoot = resolve(projectRoot, "_evalkit-results");
-let sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
+let sandboxRoot = defaultSandboxDir(projectRoot);
 let project;
 async function loadRegistry() {
 	project ??= await loadProject(projectRoot);
@@ -5336,21 +5549,20 @@ async function runEvals(options) {
 	const evaluations = options.evalIds.map((id) => registry.get(id));
 	if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
 	const provenance = await runProvenance(registry);
-	await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
+	await withFixtures(projectRoot, project?.config, (fixtures) => Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
 		const name = evaluation.name ?? authoringId(evaluation);
 		const started = performance.now();
 		return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
+			...projectRunOptions(projectRoot, project?.config),
+			...fixtures ? { fixtures } : {},
 			report: localReportStore(reportRoot),
 			workspaceRoot: sandboxRoot,
-			keepWorkspaces: project?.config?.keepSandboxes,
-			artifactExclude: project?.config?.artifacts?.exclude,
-			attachmentLimits: project?.config?.artifacts?.attachments,
 			suiteId: options.suiteId,
 			runtime: runtimeFor(evaluation),
 			concurrency: options.concurrency,
 			...provenance
 		})), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
-	}), { concurrency: options.concurrency })));
+	}), { concurrency: options.concurrency }))));
 }
 /** Detected per launch: `evalkit dev` keeps running while commits change. */
 async function runProvenance(registry) {
@@ -5408,19 +5620,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, repeat) {
 			const name = evaluation.name ?? authoringId(evaluation);
 			const started = performance.now();
 			runLog.started(name, body.parameters ?? {}, suiteId);
-			runProvenance(registry).then((provenance) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
+			runProvenance(registry).then((provenance) => withFixtures(projectRoot, project?.config, (fixtures) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
+				...projectRunOptions(projectRoot, project?.config),
+				...fixtures ? { fixtures } : {},
 				selection,
 				concurrency: 1,
 				...repeat ? { repeat } : {},
 				...suiteId ? { suiteId } : {},
 				report: localReportStore(reportRoot),
 				workspaceRoot: sandboxRoot,
-				keepWorkspaces: project?.config?.keepSandboxes,
-				artifactExclude: project?.config?.artifacts?.exclude,
-				attachmentLimits: project?.config?.artifacts?.attachments,
 				...provenance,
 				onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
-			})))).catch((error) => runLog.failed(name, error)).finally(() => {
+			}))))).catch((error) => runLog.failed(name, error)).finally(() => {
 				matrixRunning = false;
 			});
 		} else {
@@ -5515,9 +5726,9 @@ function registerReportRoutes(app, registry, reports) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
 	});
-	app.get("/v1/runs/:runId/events", async (context) => {
+	app.get("/v1/runs/:runId/transcript", async (context) => {
 		try {
-			return context.json({ events: await reports.trajectory(context.req.param("runId")) });
+			return context.json(await reports.transcript(context.req.param("runId")));
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
@@ -5550,7 +5761,7 @@ async function useProject(configPath) {
 	project = await loadProject(projectRoot, configPath);
 	projectRoot = project.root;
 	reportRoot = resolve(projectRoot, project.config.reportDir ?? "_evalkit-results");
-	sandboxRoot = resolve(projectRoot, project.config.sandboxDir ?? "_evalkit-sandbox");
+	sandboxRoot = sandboxLocation(project).path;
 }
 /**
 * `--port` is honoured exactly; without it the server takes the first free
@@ -5671,7 +5882,8 @@ async function tidyAction(options) {
 		registry: project.registry,
 		reportRoot,
 		sandboxRoot,
-		keep: project.config.keepSandboxes
+		keep: project.config.keepSandboxes,
+		...project.config.isolation?.kind === "container" ? { runtime: project.config.isolation.runtime } : {}
 	}), (value) => {
 		const keep = project.config.keepSandboxes ?? "failed";
 		if (keep === "all") return "all kept (keepSandboxes: 'all')";
@@ -5690,14 +5902,21 @@ async function upgradeAction(options) {
 	banner("upgrade", VERSION);
 	const { root, config } = await loadConfig(projectRoot, options.config);
 	reportRoot = resolve(root, config.reportDir ?? "_evalkit-results");
-	const report = await step(options.dryRun ? "Checking results" : "Upgrading results", () => upgradeReports(reportRoot, { dryRun: options.dryRun }), (value) => `${plural(value.upgraded, "run")} from an older EvalKit ${options.dryRun ? "found" : "upgraded"}`);
+	const report = await step(options.dryRun ? "Checking results" : "Upgrading results", () => upgradeReports(reportRoot, { dryRun: options.dryRun }), (value) => `${plural(value.transcripts, "run")} ${options.dryRun ? "to upgrade" : "upgraded"}`);
 	console.log("");
-	if (!report.upgraded) console.log(`  ${cyan("✓")} Every run is already in the current format.`);
-	else {
-		const into = `${plural(report.upgraded, "run")} from older EvalKit versions into ${plural(report.runs, "run")}`;
-		console.log(options.dryRun ? `  ${cyan("◆")} Would upgrade ${into}. Nothing was changed.` : `  ${cyan("✓")} Upgraded ${into}.`);
+	if (!report.upgraded && !report.transcripts && !report.failed.length) console.log(`  ${cyan("✓")} Every run is already in the current format.`);
+	if (report.upgraded) {
+		const into = `${plural(report.upgraded, "run")} with trials into ${plural(report.runs, "run")}`;
+		console.log(options.dryRun ? `  ${cyan("◆")} Would upgrade ${into}.` : `  ${cyan("✓")} Upgraded ${into}.`);
 		if (report.batches) console.log(`  ${dim(`${plural(report.batches, "run")} had more than one trial; each trial is now a run in a batch.`)}`);
 	}
+	if (report.transcripts) console.log(options.dryRun ? `  ${cyan("◆")} Would convert ${plural(report.transcripts, "trajectory", "trajectories")} to ATXF transcripts.` : `  ${cyan("✓")} Converted ${plural(report.transcripts, "trajectory", "trajectories")} to ATXF transcripts.`);
+	if (options.dryRun && (report.upgraded || report.transcripts)) console.log(`  ${dim("Nothing was changed.")}`);
+	if (report.failed.length) {
+		console.log(`  ${red("✗")} Left ${plural(report.failed.length, "run")} as ${report.failed.length === 1 ? "it was" : "they were"}; ${report.failed.length === 1 ? "it" : "they"} could not be converted:`);
+		for (const { runId, reason } of report.failed) console.log(`    ${runId}  ${dim(reason)}`);
+		process.exitCode = 1;
+	}
 	if (report.running.length) console.log(`  ${yellow("◆")} Left ${plural(report.running.length, "run")} still running alone; upgrade again when they finish.`);
 	console.log("");
 }