@leostera/evalkit 0.10.0 → 0.11.0

package/dist/cli.mjs

@@ -1,18 +1,18 @@
 #!/bin/sh
 ':' //; exec "${EVALKIT_RUNTIME:-$(command -v bun || command -v node)}" "$0" "$@"
-import { H as defineEvalMatrix, U as canonicalParameters, h as registerEvals, i as agentRuntimes, l as defineSuite, m as recordError, o as authoringId, tt as passMark } from "./src-B14fXAGp.mjs";
-import { a as readRunEvents, c as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, i as ownerIsGone, l as localReportStore, n as upgradeReports, o as readRunManifest, p as globMatcher, r as runMatrix, t as runEval, u as runWorkspacePath } from "./src-BfSQKOJP.mjs";
-import { f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, s as repoKey } from "./index-db-B1FeXzmS.mjs";
-import { t as createDirectoryShareApp } from "./src-Bccv9igQ.mjs";
+import { Et as resolveLimits, Q as canonicalParameters, St as formatLimit, Tt as parseLimits, Z as defineEvalMatrix, _ as recordError, i as agentRuntimes, l as defineSuite, lt as passMark, o as authoringId, v as registerEvals, wt as parseLimitFlag } from "./src-D6Ax2t9b.mjs";
+import { _ as removeRunContainer, a as readRunEvents, c as readRunSubagents, d as localReportStore, f as runWorkspacePath, g as fixtureSet, h as globMatcher, i as ownerIsGone, l as readRunSummary, m as staticDatasetFiles, n as upgradeReports, o as readRunManifest, p as staticDatasetFile, r as runMatrix, t as runEval, u as readRunTranscript } from "./src-Dc3GXacB.mjs";
+import { f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, s as repoKey } from "./index-db-DI7xKMSX.mjs";
+import { t as createDirectoryShareApp } from "./src-CXzbHAKN.mjs";
+import { createHash } from "node:crypto";
 import { Effect, LogLevel, Logger } from "effect";
 import { tmpdir } from "node:os";
 import { cp, lstat, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
 import path, { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
-import { createHash } from "node:crypto";
+import childProcess, { execFile, spawn } from "node:child_process";
 import fs, { existsSync } from "node:fs";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { Hono } from "hono";
-import childProcess, { execFile, spawn } from "node:child_process";
 import { promisify, stripVTControlCharacters } from "node:util";
 import { createServer } from "node:http";
 import { EventEmitter } from "node:events";
@@ -241,7 +241,7 @@ function createDashboardReportReader(root) {
 		} catch {
 			return false;
 		}
-		return typeof version === "number" && version < 6;
+		return typeof version === "number" && version < 7;
 	}
 	async function readRun(id) {
 		if (await isLegacyRun(id)) {
@@ -271,6 +271,9 @@ function createDashboardReportReader(root) {
 				manifest.aut.version
 			].filter(Boolean).join(" / ") } : {},
 			status: summary ? summary.status : interrupted ? "cancelled" : "running",
+			...manifest.limits ? { limits: manifest.limits } : {},
+			...manifest.isolation ? { isolation: manifest.isolation } : {},
+			...summary?.limit ? { limit: summary.limit } : {},
 			...interrupted ? { interrupted: true } : {},
 			startedAt: manifest.startedAt,
 			...summary?.endedAt ? { completedAt: summary.endedAt } : {},
@@ -332,9 +335,19 @@ function createDashboardReportReader(root) {
 	}
 	const candidateFile = (runId, path) => artifactFile(runId, "candidate", path);
 	const attachmentFile = (runId, name) => artifactFile(runId, "attachments", name);
-	async function trajectory(runId) {
+	/** The agent's transcript and EvalKit's events, as far as a running run has got. */
+	async function transcript(runId) {
 		runPath(root, runId);
-		return await ifPresent(() => readRunEvents(root, runId)) ?? [];
+		const [recorded, subagents, runEvents] = await Promise.all([
+			ifPresent(() => readRunTranscript(root, runId)),
+			readRunSubagents(root, runId),
+			ifPresent(() => readRunEvents(root, runId))
+		]);
+		return {
+			transcript: recorded ?? null,
+			subagents,
+			runEvents: runEvents ?? []
+		};
 	}
 	return {
 		listRuns,
@@ -345,7 +358,7 @@ function createDashboardReportReader(root) {
 		listCandidateWorkspace,
 		candidateFile,
 		attachmentFile,
-		trajectory
+		transcript
 	};
 }
 /** Both artifact endpoints walk a specific root; never broaden the candidate-file read root. */
@@ -522,7 +535,7 @@ function directorySink(root) {
 }
 /**
 * One run's snapshot, shared by `evalkit build` (files on disk) and
-* `evalkit share` (an upload bundle): `detail.json`, `events.json`,
+* `evalkit share` (an upload bundle): `detail.json`, `transcript.json`,
 * `artifacts.json` and `workspace.json`, the candidate files under
 * `workspace/`, and attachments under `attachments/` (both left out with
 * `workspaceFiles: false`; they stay listed). Evaluator workspaces are never
@@ -531,7 +544,7 @@ function directorySink(root) {
 async function snapshotRun(reports, runId, sink, options = {}) {
 	const detail = await reports.runDetail(runId);
 	await sink.json("detail.json", detail);
-	await sink.json("events.json", { events: await reports.trajectory(runId) });
+	await sink.json("transcript.json", await reports.transcript(runId));
 	await sink.json("artifacts.json", { artifacts: await reports.listArtifacts(runId) });
 	const workspace = await reports.listCandidateWorkspace(runId);
 	await sink.json("workspace.json", workspace);
@@ -979,8 +992,8 @@ function bytes(size) {
 	if (size < 1024 ** 2) return `${(size / 1024).toFixed(1)} kB`;
 	return `${(size / 1024 ** 2).toFixed(1)} MB`;
 }
-function plural(count, noun) {
-	return `${count} ${noun}${count === 1 ? "" : "s"}`;
+function plural(count, noun, nouns = `${noun}s`) {
+	return `${count} ${count === 1 ? noun : nouns}`;
 }
 /** Times one build step, showing a live line while it runs. */
 async function step(label, work, describe) {
@@ -1078,6 +1091,7 @@ async function confirm(question) {
 //#region ../cli/src/run-reporter.ts
 function runOutcome(result) {
 	if (result.status === "cancelled") return "cancelled";
+	if (result.status === "limited") return "limited";
 	return result.status === "passed" ? "passed" : "failed";
 }
 function value(score) {
@@ -1148,17 +1162,26 @@ function parameterLabel(parameters) {
 }
 /** One printed line for a finished run; failing reasons on the same line. */
 function runLine(scenario) {
-	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : red("✗");
+	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : scenario.outcome === "limited" ? yellow("✗") : red("✗");
 	const parameters = parameterLabel(scenario.parameters);
 	const batch = scenario.result.batch;
 	const position = batch ? dim(` #${batch.index + 1}/${batch.size}`) : "";
 	const details = [
-		...scenario.outcome === "failed" ? scenario.reasons : [],
+		...scenario.outcome === "limited" ? [limitReason(scenario.result)] : [],
+		...scenario.outcome === "failed" || scenario.outcome === "limited" ? scenario.reasons : [],
 		...scenario.outcome === "cancelled" ? ["cancelled"] : [],
 		...scenario.skipped.map((name) => `${name} skipped`)
 	];
 	return `  ${mark} ${bold(scenario.label)}${parameters ? ` ${dim(parameters)}` : ""}${position}${details.length ? `   ${dim(details.join(" · "))}` : ""}`;
 }
+/** `limited: 41 of 40 steps · score 0.40 (graded what it left)`. */
+function limitReason(result) {
+	const limit = result.limit;
+	if (!limit) return "limited";
+	const used = limit.limit === "time" ? `${formatLimit("time", Math.round(limit.used / 1e3) * 1e3)} of ${formatLimit("time", limit.value)}` : `${limit.used.toLocaleString("en-US")} of ${formatLimit(limit.limit, limit.value)}`;
+	const overall = result.scoring?.overall;
+	return `limited: ${used}${overall === void 0 || !result.scoring?.results.length ? "" : ` · score ${overall.toFixed(2)} (graded what it left)`}`;
+}
 /**
 * Prints run progress. On a terminal, failures print as they happen and a live
 * line shows counts and what is running. Elsewhere (CI), every finished
@@ -1203,8 +1226,15 @@ function createRunReporter(options) {
 				timer.unref?.();
 			}
 		},
-		onEvent(run, event) {
-			if (event.source === "runner" && event.kind === "run-started") {
+		onUpdate(run, update) {
+			if (update.kind === "record") {
+				const activity = running.get(run.runId);
+				const call = update.record.type === "message" && update.record.role === "assistant" ? update.record.content.filter((block) => block.type === "tool_call").at(-1) : void 0;
+				if (activity && call?.type === "tool_call") activity.doing = `tool ${call.name}`;
+				return;
+			}
+			const event = update.event;
+			if (event.kind === "run-started") {
 				running.set(run.runId, {
 					label: options.labelFor(run),
 					startedAt: now()
@@ -1213,10 +1243,10 @@ function createRunReporter(options) {
 			}
 			const activity = running.get(run.runId);
 			if (!activity) return;
-			if (event.source === "runner" && (event.kind === "run-completed" || event.kind === "error")) running.delete(run.runId);
-			else if (event.source === "aut" && event.kind === "tool-call") activity.doing = `tool ${event.name}`;
-			else if (event.source === "runner" && event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
-			else if (event.source === "runner" && event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
+			if (event.kind === "run-completed" || event.kind === "error") running.delete(run.runId);
+			else if (event.kind === "limit-reached") activity.doing = `stopping at its ${event.limit} limit`;
+			else if (event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
+			else if (event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
 		},
 		onRun(scenario) {
 			scenarios.push(scenario);
@@ -1233,6 +1263,7 @@ function createRunReporter(options) {
 			return {
 				passed: scenarios.filter((s) => s.outcome === "passed").length,
 				failed: scenarios.filter((s) => s.outcome === "failed").length,
+				limited: scenarios.filter((s) => s.outcome === "limited").length,
 				cancelled: scenarios.filter((s) => s.outcome === "cancelled").length,
 				runs: scenarios.length,
 				scenarios: new Set(scenarios.map((s) => `${s.label} ${parameterLabel(s.parameters)}`)).size,
@@ -1246,6 +1277,7 @@ function totalsLine(totals) {
 	const parts = [
 		green(`${totals.passed} passed`),
 		totals.failed ? red(`${totals.failed} failed`) : "",
+		totals.limited ? yellow(`${totals.limited} limited`) : "",
 		totals.cancelled ? yellow(`${totals.cancelled} cancelled`) : ""
 	].filter(Boolean);
 	const coverage = totals.runs === totals.scenarios ? plural(totals.runs, "run") : `${plural(totals.runs, "run")} of ${plural(totals.scenarios, "scenario")}`;
@@ -1269,13 +1301,17 @@ function junitReport(scenarios) {
 		bySuite.set(suite, [...bySuite.get(suite) ?? [], scenario]);
 	}
 	const suites = [...bySuite].map(([suite, cases]) => {
-		const failures = cases.filter((c) => c.outcome === "failed").length;
+		const failures = cases.filter((c) => c.outcome === "failed" || c.outcome === "limited").length;
 		const skipped = cases.filter((c) => c.outcome === "cancelled" || c.outcome === "passed" && c.skipped.length > 0 && allSkipped(c)).length;
 		const time = cases.reduce((sum, c) => sum + (c.durationMs ?? 0), 0);
 		const testcases = cases.map((scenario) => {
 			const parameters = parameterLabel(scenario.parameters);
 			const batch = scenario.result.batch;
 			const attributes = `name="${escape((parameters ? `${scenario.evalId} [${parameters}]` : scenario.evalId) + (batch ? ` #${batch.index + 1}` : ""))}" classname="${escape(suite)}" time="${seconds(scenario.durationMs)}"`;
+			if (scenario.outcome === "limited") {
+				const reasons = [limitReason(scenario.result), ...scenario.reasons];
+				return `    <testcase ${attributes}>\n      <failure message="${escape(reasons.join("; "))}" type="limited">${escape(reasons.join("\n"))}</failure>\n    </testcase>`;
+			}
 			if (scenario.outcome === "failed") return `    <testcase ${attributes}>\n      <failure message="${escape(scenario.reasons.join("; ") || "failed")}">${escape(scenario.reasons.join("\n"))}</failure>\n    </testcase>`;
 			if (scenario.outcome === "cancelled") return `    <testcase ${attributes}>\n      <skipped message="cancelled"/>\n    </testcase>`;
 			if (allSkipped(scenario)) return `    <testcase ${attributes}>\n      <skipped message="${escape(`all scorers skipped: ${scenario.skipped.join(", ")}`)}"/>\n    </testcase>`;
@@ -1283,7 +1319,7 @@ function junitReport(scenarios) {
 		});
 		return `  <testsuite name="${escape(suite)}" tests="${cases.length}" failures="${failures}" errors="0" skipped="${skipped}" time="${seconds(time)}">\n${testcases.join("\n")}\n  </testsuite>`;
 	});
-	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed").length}">\n${suites.join("\n")}\n</testsuites>\n`;
+	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed" || c.outcome === "limited").length}">\n${suites.join("\n")}\n</testsuites>\n`;
 }
 /** True when every score was skip()ped. */
 function allSkipped(scenario) {
@@ -1291,9 +1327,71 @@ function allSkipped(scenario) {
 	return results.length > 0 && results.every((score) => score.skipped);
 }
 //#endregion
+//#region ../cli/src/run-options.ts
+/**
+* What every run of the project shares, from its config, wherever it is
+* launched: the CLI's `run` and the dashboard's buttons.
+*/
+function projectRunOptions(root, config) {
+	return {
+		root,
+		...config?.isolation ? { isolation: config.isolation } : {},
+		limits: { base: parseLimits(config?.limits, "The config") },
+		...config?.keepSandboxes ? { keepWorkspaces: config.keepSandboxes } : {},
+		...config?.artifacts?.exclude ? { artifactExclude: config.artifacts.exclude } : {},
+		...config?.artifacts?.attachments ? { attachmentLimits: config.artifacts.attachments } : {},
+		...config?.judge ? { judge: config.judge } : {}
+	};
+}
+/**
+* The project's fixtures for one invocation, or undefined when it has
+* none. The caller stops them when its runs are done.
+*/
+function projectFixtures(root, config) {
+	const fixtures = config?.fixtures;
+	if (!fixtures || !Object.keys(fixtures).length) return void 0;
+	return fixtureSet(fixtures, { root });
+}
+/** Runs `work` with the project's fixtures, stopping them afterwards. */
+async function withFixtures(root, config, work) {
+	const fixtures = projectFixtures(root, config);
+	try {
+		return await work(fixtures);
+	} finally {
+		await fixtures?.stop();
+	}
+}
+//#endregion
+//#region ../cli/src/package-manager.ts
+const LOCKFILES = [
+	["bun.lock", "bun"],
+	["bun.lockb", "bun"],
+	["pnpm-lock.yaml", "pnpm"],
+	["yarn.lock", "yarn"],
+	["package-lock.json", "npm"]
+];
+/**
+* The package manager a project uses: the one that started this process
+* (`npm_config_user_agent`), else the nearest lockfile at or above `root`
+* (a workspace keeps it at its root), else bun. Suggesting another manager's
+* command can make someone run its install and rewrite `node_modules`.
+*/
+function detectPackageManager(root, userAgent = process.env.npm_config_user_agent) {
+	const started = userAgent?.split("/")[0];
+	if (started === "bun" || started === "pnpm" || started === "yarn" || started === "npm") return started;
+	for (let dir = root;; dir = dirname(dir)) {
+		for (const [file, manager] of LOCKFILES) if (existsSync(join(dir, file))) return manager;
+		if (dirname(dir) === dir) return "bun";
+	}
+}
+/** `pnpm run evalkit dev` (or `npm run evalkit -- dev`: npm needs the `--` for flags). */
+function evalkitCommand(manager, args) {
+	return manager === "npm" ? `npm run evalkit -- ${args}` : `${manager} run evalkit ${args}`;
+}
+//#endregion
 //#region ../cli/src/version.ts
 /** The CLI version; kept in sync with package.json when releasing. */
-const VERSION = "0.10.0";
+const VERSION = "0.11.0";
 //#endregion
 //#region ../cli/src/run-command.ts
 /** Validates `run` flags into a selection, parameter overrides and limits. */
@@ -1306,14 +1404,7 @@ function normalizeRunOptions(values, positionals) {
 		if (!Number.isSafeInteger(number) || number < 1) throw new Error(`--${name} must be a positive integer`);
 		return number;
 	}
-	for (const [flag, key] of [
-		["max-tokens", "maxTokens"],
-		["chat-timeout-ms", "chatTimeoutMs"],
-		["turn-budget", "turnBudget"]
-	]) {
-		const value = integer(flag);
-		if (value !== void 0) parameters[key] = value;
-	}
+	const limits = resolveLimits(...(values.limit ?? []).flatMap((value) => value.split(",")).map(parseLimitFlag));
 	const selection = {};
 	for (const key of ["model", "mode"]) if (values[key]?.length) selection[key] = values[key].flatMap((value) => value.split(","));
 	for (const value of values.select ?? []) {
@@ -1339,6 +1430,7 @@ function normalizeRunOptions(values, positionals) {
 		positionals,
 		parameters,
 		selection,
+		limits,
 		concurrency: integer("concurrency"),
 		repeat: integer("repeat")
 	};
@@ -1411,6 +1503,7 @@ async function runProjectCommand(cwd, parsed) {
 	const cells = matrix.count(selection);
 	if (!cells) throw new Error("Selection contains no eligible cells; check cases and exclusions");
 	const repeat = parsed.repeat ?? project.config.execution?.repeat;
+	const configLimits = parseLimits(project.config.limits, "The config");
 	let runs = 0;
 	for (const cell of matrix.cells(selection)) runs += repeat ?? cell.eval.repeat ?? 1;
 	const plan = {
@@ -1421,6 +1514,12 @@ async function runProjectCommand(cwd, parsed) {
 		parameters: selection.parameters,
 		overrides: selection.overrides,
 		repeat: repeat ?? "scenario",
+		limits: {
+			config: configLimits,
+			cli: parsed.limits
+		},
+		isolation: project.config.isolation ?? { kind: "none" },
+		fixtures: Object.keys(project.config.fixtures ?? {}),
 		scoring: Object.fromEntries(registry.catalog({ root: project.root }).filter((entry) => matrix.evals.some((definition) => definition.id === entry.id)).map((entry) => [entry.id, plannedChecks(entry)]))
 	};
 	if (values["dry-run"]) {
@@ -1462,7 +1561,9 @@ async function runProjectCommand(cwd, parsed) {
 	reporter.start();
 	let summary;
 	try {
-		summary = await Effect.runPromise(runnerLogs(runMatrix(matrix, {
+		summary = await withFixtures(project.root, project.config, (fixtures) => Effect.runPromise(runnerLogs(runMatrix(matrix, {
+			...projectRunOptions(project.root, project.config),
+			...fixtures ? { fixtures } : {},
 			selection,
 			concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
 			...repeat ? { repeat } : {},
@@ -1473,12 +1574,12 @@ async function runProjectCommand(cwd, parsed) {
 			catalog: runCatalog(registry, project.root),
 			report: localReportStore(reportDir),
 			workspaceRoot: sandboxDir,
-			keepWorkspaces: project.config.keepSandboxes,
-			artifactExclude: project.config.artifacts?.exclude,
-			attachmentLimits: project.config.artifacts?.attachments,
-			judge: project.config.judge,
 			signal: controller.signal,
-			onEvent: (run, event) => reporter.onEvent(run, event),
+			limits: {
+				base: configLimits,
+				override: parsed.limits
+			},
+			onUpdate: (run, update) => reporter.onUpdate(run, update),
 			onResult: (cell, results) => {
 				for (const result of results) {
 					const report = runReport(cell, result, suiteFor(authoringId(cell.eval)));
@@ -1494,7 +1595,7 @@ async function runProjectCommand(cwd, parsed) {
 					reporter.onRun(report);
 				}
 			}
-		})));
+		}))));
 	} finally {
 		process.off("SIGINT", onInterrupt);
 	}
@@ -1508,12 +1609,14 @@ async function runProjectCommand(cwd, parsed) {
 	else {
 		console.log(`\n${totalsLine(totals)}`);
 		if (values.junit) console.log(`  ${dim(`JUnit report: ${values.junit}`)}`);
-		console.log(`  ${cyan("➜")} bun run evalkit dev        ${dim("to inspect")}`);
-		if (project.config.share) console.log(`  ${cyan("➜")} bun run evalkit share      ${dim("to publish")}`);
+		const manager = detectPackageManager(project.root);
+		const hint = (args, why) => console.log(`  ${cyan("➜")} ${evalkitCommand(manager, args).padEnd(24)} ${dim(why)}`);
+		hint("dev", "to inspect");
+		if (project.config.share) hint("share", "to publish");
 		console.log("");
 	}
 	if (controller.signal.aborted) process.exitCode = 130;
-	else if (summary.failed || broken.length) process.exitCode = 1;
+	else if (summary.failed || summary.limited || broken.length) process.exitCode = 1;
 }
 /** A scenario's checkpoints and final scorers, as `--dry-run` lists them. */
 function plannedChecks(entry) {
@@ -1582,8 +1685,8 @@ async function tidyReports(options) {
 	return result;
 }
 /**
-* Remove the run workspaces `keep` would not have kept: those of passed runs
-* by default. Unfinished runs are never touched, so a run in progress keeps
+* Remove the run workspaces `keep` would not have kept, and their kept
+* containers: those of passed runs by default. Unfinished runs are never touched, so a run in progress keeps
 * its workspace; workspaces without a report are left alone.
 */
 async function tidySandboxes(options) {
@@ -1600,6 +1703,7 @@ async function tidySandboxes(options) {
 			recursive: true,
 			force: true
 		});
+		if (run.isolation?.kind === "container") await removeRunContainer(run.id, options.runtime);
 		removed += 1;
 	}
 	return { removed };
@@ -1678,7 +1782,7 @@ async function runBundle(options) {
 	files.push({
 		path: "run.json",
 		data: encode({
-			snapshotVersion: 2,
+			snapshotVersion: 3,
 			...options.run
 		})
 	});
@@ -1873,7 +1977,7 @@ export const greetingAgent = defineAgent({
   identity: { id: 'greeting-agent', kind: 'in-process' },
   async start({ context }) {
     return {
-      // Return the reply; EvalKit records it. Call onEvent instead to stream events.
+      // Return the reply; EvalKit records it. Use the transcript writer in start() to record tool calls and usage.
       async send(message: string) {
         const file = join(context.workspace.root, 'greeting', 'greeting.txt');
         const prefix = (await readFile(file, 'utf8')).trim();
@@ -1940,7 +2044,7 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
 }
 //#endregion
 //#region ../cli/src/new-project.ts
-const npmDependency = "^0.10.0";
+const npmDependency = "^0.11.0";
 const ignored = [
 	"node_modules/",
 	"_evalkit-results/",
@@ -2055,9 +2159,11 @@ async function writeStarterReadme(root, name, inPlace) {
 }
 //#endregion
 //#region ../cli/src/compare.ts
+/** Runs that ended on their own; a limited run is a finished failure. */
 const FINISHED = /* @__PURE__ */ new Set([
 	"passed",
 	"failed",
+	"limited",
 	"errored"
 ]);
 function unitKey(run) {
@@ -5364,12 +5470,13 @@ function createProgram(actions) {
 		showGlobalOptions: false
 	}).addHelpText("after", `\nRun from your eval project's directory. Docs: https://evalkit.leostera.dev/docs/\n`);
 	program.command("new").description("create an eval project (never overwrites files)").argument("<directory>", "where to create it; use . for the current directory").addHelpText("after", examples(["bunx @leostera/evalkit new ./agent-evals", "evalkit new ."])).action((directory) => actions.create(directory));
-	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run the scenarios in this suite; with -m, only those in both").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("-a, --select <axis=value>", "keep only these values of a matrix axis (a for axis); repeatable, comma-separated", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --repeat <n>", "runs of each scenario, as one batch; overrides config and scenario").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--max-tokens <n>", "pass maxTokens to the agent in context.parameters").option("--turn-budget <n>", "pass turnBudget to the agent in context.parameters").option("--chat-timeout-ms <n>", "pass chatTimeoutMs to the agent in context.parameters").option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
+	program.command("run").description("run scenarios and save the results in _evalkit-results/").argument("[eval-ids...]", "scenarios to run (default: all)").addOption(configOption()).option("-e, --eval <id>", "run this scenario; repeatable, or comma-separated", collect).option("-s, --suite <id>", "run the scenarios in this suite; with -m, only those in both").option("-m, --matrix <id>", "run this matrix").option("--model <value>", "--select model=<value> for each comma-separated value", collect).option("--mode <value>", "--select mode=<value> for each comma-separated value", collect).option("-a, --select <axis=value>", "keep only these values of a matrix axis (a for axis); repeatable, comma-separated; one value per axis runs one cell", collect).option("--param <name=json>", "set a parameter for every scenario; repeatable", collect).option("-n, --repeat <n>", "runs of each scenario, as one batch; overrides config and scenario").option("-j, --concurrency <n>", "matrix cells to run at once (default: 4)").option("--limit <kind=value>", "stop each run past this limit: steps=80, tokens=2000000, time=20m; repeatable; overrides config, scenario and matrix", collect).option("--local", "require the agent's local runtime").option("--all", "allow more matrix cells than execution.maxCells").option("--dry-run", "print the plan as JSON; run nothing").option("--json", "print one JSON result per matrix cell").option("--junit <file>", "write a JUnit XML report").option("--skip-broken", "run healthy scenarios when some files fail to load, then exit 1").option("-v, --verbose", "print every scenario, not only failures").addOption(logLevelOption()).addHelpText("after", examples([
 		"evalkit run",
 		"evalkit run greeting -n 5",
 		"evalkit run -s billing --junit _evalkit-results/junit.xml",
 		"evalkit run -m models --model model-a,model-b --dry-run",
-		"evalkit run -m models -s testing -a model=luna -a toolchain=riot"
+		"evalkit run -m models -s testing -a model=luna -a toolchain=riot",
+		"evalkit run todo-app -a model=luna -a skills=none -a docs=search"
 	])).action((evalIds, options) => {
 		setRunnerLogLevel(options.logLevel);
 		return actions.run(runInput(evalIds, options));
@@ -5408,9 +5515,7 @@ function runInput(evalIds, flags) {
 		mode: flags.mode,
 		select: flags.select,
 		param: flags.param,
-		"max-tokens": flags.maxTokens,
-		"chat-timeout-ms": flags.chatTimeoutMs,
-		"turn-budget": flags.turnBudget,
+		limit: flags.limit,
 		concurrency: flags.concurrency,
 		repeat: flags.repeat,
 		json: flags.json,
@@ -5444,22 +5549,20 @@ async function runEvals(options) {
 	const evaluations = options.evalIds.map((id) => registry.get(id));
 	if (evaluations.some((e) => !e)) throw new Error("Unknown eval");
 	const provenance = await runProvenance(registry);
-	await Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
+	await withFixtures(projectRoot, project?.config, (fixtures) => Effect.runPromise(runnerLogs(Effect.all(evaluations.map((evaluation) => {
 		const name = evaluation.name ?? authoringId(evaluation);
 		const started = performance.now();
 		return Effect.sync(() => runLog.started(name, {}, options.suiteId)).pipe(Effect.andThen(runEval(evaluation, {
+			...projectRunOptions(projectRoot, project?.config),
+			...fixtures ? { fixtures } : {},
 			report: localReportStore(reportRoot),
 			workspaceRoot: sandboxRoot,
-			keepWorkspaces: project?.config?.keepSandboxes,
-			artifactExclude: project?.config?.artifacts?.exclude,
-			attachmentLimits: project?.config?.artifacts?.attachments,
-			judge: project?.config?.judge,
 			suiteId: options.suiteId,
 			runtime: runtimeFor(evaluation),
 			concurrency: options.concurrency,
 			...provenance
 		})), Effect.tap((result) => runLog.finished(name, {}, result, performance.now() - started)), Effect.tapError((error) => Effect.sync(() => runLog.failed(name, error))));
-	}), { concurrency: options.concurrency })));
+	}), { concurrency: options.concurrency }))));
 }
 /** Detected per launch: `evalkit dev` keeps running while commits change. */
 async function runProvenance(registry) {
@@ -5517,20 +5620,18 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, repeat) {
 			const name = evaluation.name ?? authoringId(evaluation);
 			const started = performance.now();
 			runLog.started(name, body.parameters ?? {}, suiteId);
-			runProvenance(registry).then((provenance) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
+			runProvenance(registry).then((provenance) => withFixtures(projectRoot, project?.config, (fixtures) => Effect.runPromise(runnerLogs(runMatrix(dashboardMatrix, {
+				...projectRunOptions(projectRoot, project?.config),
+				...fixtures ? { fixtures } : {},
 				selection,
 				concurrency: 1,
 				...repeat ? { repeat } : {},
 				...suiteId ? { suiteId } : {},
 				report: localReportStore(reportRoot),
 				workspaceRoot: sandboxRoot,
-				keepWorkspaces: project?.config?.keepSandboxes,
-				artifactExclude: project?.config?.artifacts?.exclude,
-				attachmentLimits: project?.config?.artifacts?.attachments,
-				judge: project?.config?.judge,
 				...provenance,
 				onResult: (cell, result) => runLog.cell(cell, result, performance.now() - started)
-			})))).catch((error) => runLog.failed(name, error)).finally(() => {
+			}))))).catch((error) => runLog.failed(name, error)).finally(() => {
 				matrixRunning = false;
 			});
 		} else {
@@ -5625,9 +5726,9 @@ function registerReportRoutes(app, registry, reports) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
 	});
-	app.get("/v1/runs/:runId/events", async (context) => {
+	app.get("/v1/runs/:runId/transcript", async (context) => {
 		try {
-			return context.json({ events: await reports.trajectory(context.req.param("runId")) });
+			return context.json(await reports.transcript(context.req.param("runId")));
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Not found", 404);
 		}
@@ -5781,7 +5882,8 @@ async function tidyAction(options) {
 		registry: project.registry,
 		reportRoot,
 		sandboxRoot,
-		keep: project.config.keepSandboxes
+		keep: project.config.keepSandboxes,
+		...project.config.isolation?.kind === "container" ? { runtime: project.config.isolation.runtime } : {}
 	}), (value) => {
 		const keep = project.config.keepSandboxes ?? "failed";
 		if (keep === "all") return "all kept (keepSandboxes: 'all')";
@@ -5800,14 +5902,21 @@ async function upgradeAction(options) {
 	banner("upgrade", VERSION);
 	const { root, config } = await loadConfig(projectRoot, options.config);
 	reportRoot = resolve(root, config.reportDir ?? "_evalkit-results");
-	const report = await step(options.dryRun ? "Checking results" : "Upgrading results", () => upgradeReports(reportRoot, { dryRun: options.dryRun }), (value) => `${plural(value.upgraded, "run")} from an older EvalKit ${options.dryRun ? "found" : "upgraded"}`);
+	const report = await step(options.dryRun ? "Checking results" : "Upgrading results", () => upgradeReports(reportRoot, { dryRun: options.dryRun }), (value) => `${plural(value.transcripts, "run")} ${options.dryRun ? "to upgrade" : "upgraded"}`);
 	console.log("");
-	if (!report.upgraded) console.log(`  ${cyan("✓")} Every run is already in the current format.`);
-	else {
-		const into = `${plural(report.upgraded, "run")} from older EvalKit versions into ${plural(report.runs, "run")}`;
-		console.log(options.dryRun ? `  ${cyan("◆")} Would upgrade ${into}. Nothing was changed.` : `  ${cyan("✓")} Upgraded ${into}.`);
+	if (!report.upgraded && !report.transcripts && !report.failed.length) console.log(`  ${cyan("✓")} Every run is already in the current format.`);
+	if (report.upgraded) {
+		const into = `${plural(report.upgraded, "run")} with trials into ${plural(report.runs, "run")}`;
+		console.log(options.dryRun ? `  ${cyan("◆")} Would upgrade ${into}.` : `  ${cyan("✓")} Upgraded ${into}.`);
 		if (report.batches) console.log(`  ${dim(`${plural(report.batches, "run")} had more than one trial; each trial is now a run in a batch.`)}`);
 	}
+	if (report.transcripts) console.log(options.dryRun ? `  ${cyan("◆")} Would convert ${plural(report.transcripts, "trajectory", "trajectories")} to ATXF transcripts.` : `  ${cyan("✓")} Converted ${plural(report.transcripts, "trajectory", "trajectories")} to ATXF transcripts.`);
+	if (options.dryRun && (report.upgraded || report.transcripts)) console.log(`  ${dim("Nothing was changed.")}`);
+	if (report.failed.length) {
+		console.log(`  ${red("✗")} Left ${plural(report.failed.length, "run")} as ${report.failed.length === 1 ? "it was" : "they were"}; ${report.failed.length === 1 ? "it" : "they"} could not be converted:`);
+		for (const { runId, reason } of report.failed) console.log(`    ${runId}  ${dim(reason)}`);
+		process.exitCode = 1;
+	}
 	if (report.running.length) console.log(`  ${yellow("◆")} Left ${plural(report.running.length, "run")} still running alone; upgrade again when they finish.`);
 	console.log("");
 }