@leostera/evalkit 0.4.0 → 0.6.0

package/dist/cli.mjs

@@ -1,14 +1,15 @@
-#!/usr/bin/env bun
-import { I as defineEvalMatrix, L as canonicalParameters, m as registerEvals, n as authoringId } from "./src-ButXPY8P.mjs";
-import { a as readTrialEvents, c as readTrialSummary, i as readRunSummary, l as localReportStore, n as runMatrix, o as readTrialManifest, r as readRunManifest, t as runEval } from "./src-DoPs9Yug.mjs";
+#!/bin/sh
+':' //; exec "${EVALKIT_RUNTIME:-$(command -v bun || command -v node)}" "$0" "$@"
+import { L as defineEvalMatrix, R as canonicalParameters, d as recordError, f as registerEvals, i as authoringId, r as agentRuntimes, s as defineSuite } from "./src-D4tCwxNb.mjs";
+import { a as readRunSummary, d as staticDatasetFile, f as staticDatasetFiles, i as readRunManifest, l as readTrialSummary, n as runMatrix, o as readTrialEvents, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore } from "./src-CVSCn3bZ.mjs";
 import { c as dashboardIndex, d as encodeBundle, o as repoKey } from "./index-db-vbqEGFaU.mjs";
-import { t as createDirectoryShareApp } from "./src-DzZueFn3.mjs";
+import { t as createDirectoryShareApp } from "./src-CZnDXP1K.mjs";
 import * as Schema from "effect/Schema";
 import { Effect, LogLevel, Logger, Option } from "effect";
+import * as OS from "node:os";
 import { cp, mkdir, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises";
 import * as Path from "node:path";
 import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
-import * as OS from "node:os";
 import * as Crypto from "node:crypto";
 import * as NFS from "node:fs";
 import { existsSync } from "node:fs";
@@ -67,9 +68,10 @@ import * as FiberSet from "effect/FiberSet";
 import "effect/Fiber";
 import "effect/Pool";
 import * as MutableRef from "effect/MutableRef";
+import { createServer } from "node:http";
 import { parseArgs, promisify } from "node:util";
 import * as ChildProcess from "node:child_process";
-import { execFile } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
 import * as RcRef from "effect/RcRef";
 import * as readline from "node:readline";
 import { Hono } from "hono";
@@ -5792,7 +5794,7 @@ const merge = (map1, map2) => {
 	}
 	return map1;
 };
-const escape = (string) => escapeSingleQuoted(string.replaceAll("\\", "\\\\")).replaceAll("[", "\\[").replaceAll("]", "\\]").replaceAll(":", "\\:").replaceAll("$", "\\$").replaceAll("`", "\\`").replaceAll("(", "\\(").replaceAll(")", "\\)");
+const escape$1 = (string) => escapeSingleQuoted(string.replaceAll("\\", "\\\\")).replaceAll("[", "\\[").replaceAll("]", "\\]").replaceAll(":", "\\:").replaceAll("$", "\\$").replaceAll("`", "\\`").replaceAll("(", "\\(").replaceAll(")", "\\)");
 const getShortDescription$1 = (self) => {
 	switch (self._tag) {
 		case "Empty":
@@ -5860,7 +5862,7 @@ const getZshCompletions$1 = (self, state = {
 			const possibleValues = getZshCompletions$3(self.primitiveType);
 			const multiple = state.multiple ? "*" : "";
 			const conflicts = Arr.isNonEmptyReadonlyArray(state.conflicts) ? `(${Arr.join(state.conflicts, " ")})` : "";
-			return Arr.map(names, (name) => `${conflicts}${multiple}${name}[${escape(description)}]${possibleValues}`);
+			return Arr.map(names, (name) => `${conflicts}${multiple}${name}[${escape$1(description)}]${possibleValues}`);
 		}
 		case "KeyValueMap": return getZshCompletions$1(self.argumentOption, {
 			...state,
@@ -8069,8 +8071,193 @@ const layerManager = /* @__PURE__ */ Layer.provide(layerManager$2, layerWorker);
 */
 const layer = /*#__PURE__*/ pipe(/*#__PURE__*/ Layer.mergeAll(layer$3, layer$5, layer$1, layerManager), /*#__PURE__*/ Layer.provideMerge(layer$7));
 //#endregion
+//#region ../cli/src/dataset-files.ts
+/** The project's datasets by ID; the first definition wins when IDs repeat. */
+function projectDatasets(registry) {
+	const byId = /* @__PURE__ */ new Map();
+	for (const evaluation of registry.evals) for (const dataset of evaluation.datasets ?? []) if (!byId.has(dataset.id)) byId.set(dataset.id, dataset);
+	return byId;
+}
+async function datasetListing(dataset, root) {
+	const files = await staticDatasetFiles(dataset, root);
+	return files ? {
+		kind: "static",
+		mount: dataset.mount,
+		files
+	} : {
+		kind: "computed",
+		mount: dataset.mount
+	};
+}
+//#endregion
+//#region ../cli/src/source.ts
+const exec = promisify(execFile);
+/**
+* Normalize a git remote to a stable repository key: lowercase host, no
+* credentials, port, scheme, `.git`, or trailing slash.
+*
+*   git@GitLab.example.com:group/sub/repo.git → gitlab.example.com/group/sub/repo
+*   https://user:token@github.com/o/r/        → github.com/o/r
+*   /srv/repos/thing.git (no host)            → local/thing
+*/
+function normalizeRepoUrl(url) {
+	const trimmed = url.trim().replace(/\/+$/, "").replace(/\.git$/, "");
+	const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(trimmed);
+	if (scp && !/^[a-z]+:\/\//i.test(trimmed)) return `${scp[1].toLowerCase()}/${scp[2].replace(/^\/+/, "")}`;
+	try {
+		const parsed = new URL(trimmed);
+		if (parsed.protocol !== "file:" && parsed.hostname) return `${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/+$/, "")}`;
+	} catch {}
+	return `local/${basename(trimmed)}`;
+}
+async function git(cwd, ...args) {
+	try {
+		const { stdout } = await exec("git", [
+			"-C",
+			cwd,
+			...args
+		], { timeout: 1e4 });
+		return stdout.trim() || void 0;
+	} catch {
+		return;
+	}
+}
+const posix = (path) => path.split(sep).join("/");
+const projectPath = (root, projectRoot) => root ? posix(relative(root, projectRoot)) : "";
+function ciSource(env, projectRoot) {
+	if (env.EVALKIT_SOURCE_REPO && env.EVALKIT_SOURCE_COMMIT) return {
+		repo: normalizeRepoUrl(env.EVALKIT_SOURCE_REPO),
+		project: env.EVALKIT_SOURCE_PROJECT ?? "",
+		commit: env.EVALKIT_SOURCE_COMMIT,
+		...env.EVALKIT_SOURCE_REF ? { ref: env.EVALKIT_SOURCE_REF } : {},
+		...env.EVALKIT_SOURCE_COMMITTED_AT ? { committedAt: env.EVALKIT_SOURCE_COMMITTED_AT } : {}
+	};
+	if (env.GITLAB_CI === "true" && env.CI_PROJECT_URL && env.CI_COMMIT_SHA) return {
+		repo: normalizeRepoUrl(env.CI_PROJECT_URL),
+		project: projectPath(env.CI_PROJECT_DIR, projectRoot),
+		commit: env.CI_COMMIT_SHA,
+		...env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME ? { ref: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME } : {},
+		...env.CI_COMMIT_TIMESTAMP ? { committedAt: env.CI_COMMIT_TIMESTAMP } : {},
+		ci: {
+			provider: "gitlab",
+			...env.CI_PIPELINE_ID ? { pipeline: env.CI_PIPELINE_ID } : {},
+			...env.CI_JOB_ID ? { job: env.CI_JOB_ID } : {},
+			...env.CI_JOB_URL ? { url: env.CI_JOB_URL } : {}
+		}
+	};
+	if (env.GITHUB_ACTIONS === "true" && env.GITHUB_REPOSITORY && env.GITHUB_SHA) {
+		const server = env.GITHUB_SERVER_URL ?? "https://github.com";
+		return {
+			repo: normalizeRepoUrl(`${server}/${env.GITHUB_REPOSITORY}`),
+			project: projectPath(env.GITHUB_WORKSPACE, projectRoot),
+			commit: env.GITHUB_SHA,
+			...env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME ? { ref: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME } : {},
+			ci: {
+				provider: "github",
+				...env.GITHUB_RUN_ID ? { pipeline: env.GITHUB_RUN_ID } : {},
+				...env.GITHUB_JOB ? { job: env.GITHUB_JOB } : {},
+				...env.GITHUB_RUN_ID ? { url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` } : {}
+			}
+		};
+	}
+}
+/**
+* Determine which repository and commit this project's eval code comes from.
+* Order: EVALKIT_SOURCE_* variables, GitLab/GitHub CI variables, then git.
+* Returns undefined outside git with no CI variables ("untracked" results).
+*
+* `outputDirs` (results, sandbox, snapshot) never make the tree dirty, even
+* when a project forgot to ignore them.
+*/
+async function detectSource(givenRoot, options = {}) {
+	const env = options.env ?? process.env;
+	const projectRoot = await realpath(givenRoot).catch(() => givenRoot);
+	const top = await git(projectRoot, "rev-parse", "--show-toplevel");
+	const excludes = top ? (options.outputDirs ?? []).map((dir) => join(projectRoot, relative(givenRoot, dir))).map((dir) => posix(relative(top, dir))).filter((dir) => dir && !dir.startsWith("..")).map((dir) => `:(exclude)${dir}`) : [];
+	const dirty = top ? Boolean(await git(top, "status", "--porcelain", "--", ".", ...excludes)) : false;
+	const fromCi = ciSource(env, projectRoot);
+	if (fromCi) return {
+		...fromCi,
+		...top && !env.EVALKIT_SOURCE_PROJECT ? { project: projectPath(top, projectRoot) } : {},
+		dirty
+	};
+	if (!top) return void 0;
+	const commit = await git(top, "rev-parse", "HEAD");
+	if (!commit) return void 0;
+	const remote = await git(top, "config", "--get", "remote.origin.url") ?? (await git(top, "remote"))?.split("\n").map((name) => name.trim()).find(Boolean);
+	const remoteUrl = remote && !remote.includes("/") && !remote.includes(":") ? await git(top, "config", "--get", `remote.${remote}.url`) : remote;
+	const ref = await git(top, "symbolic-ref", "--short", "-q", "HEAD");
+	const committedAt = await git(top, "show", "-s", "--format=%cI", "HEAD");
+	return {
+		repo: normalizeRepoUrl(remoteUrl ?? top),
+		project: projectPath(top, projectRoot),
+		commit,
+		...ref ? { ref } : {},
+		...committedAt ? { committedAt } : {},
+		dirty
+	};
+}
+/** What the dashboard's header shows: repository, commit, branch, dirty. */
+function projectInfo(source) {
+	return {
+		repo: source.repo,
+		...source.project ? { project: source.project } : {},
+		commit: source.commit,
+		...source.ref ? { ref: source.ref } : {},
+		dirty: source.dirty
+	};
+}
+/** The project's evals and suites, saved with each run (`catalog.json`). */
+function runCatalog(registry, root) {
+	return {
+		catalog: registry.catalog({ root }),
+		evals: registry.metadata(),
+		suites: registry.suiteMetadata()
+	};
+}
+/** `abc1234` or `abc1234+dirty`. */
+function shortSource(source) {
+	return `${source.commit.slice(0, 7)}${source.dirty ? "+dirty" : ""}`;
+}
+//#endregion
 //#region ../cli/src/dashboard-reports.ts
 const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+function runScorers(trials) {
+	const byKey = /* @__PURE__ */ new Map();
+	for (const trial of trials) for (const score of trial?.scoring?.results ?? []) {
+		const kind = score.kind;
+		const key = `${kind}\0${score.name}`;
+		const entry = byKey.get(key) ?? {
+			name: score.name,
+			kind,
+			count: 0,
+			skipped: 0,
+			sum: 0
+		};
+		if (score.skipped) entry.skipped += 1;
+		else if (score.value !== void 0) {
+			entry.count += 1;
+			entry.sum += score.value;
+		}
+		byKey.set(key, entry);
+	}
+	return [...byKey.values()].map(({ sum, ...scorer }) => ({
+		...scorer,
+		...scorer.count ? { mean: sum / scorer.count } : {}
+	}));
+}
+function runDatasets(manifests) {
+	const byId = /* @__PURE__ */ new Map();
+	for (const manifest of manifests) for (const dataset of manifest?.datasets ?? []) {
+		const digests = byId.get(dataset.id) ?? /* @__PURE__ */ new Set();
+		digests.add(dataset.digest);
+		byId.set(dataset.id, digests);
+	}
+	return [...byId].map(([id, digests]) => ({
+		id,
+		digests: [...digests].sort()
+	}));
+}
 /** Written into a run directory by `evalkit tidy`; deleting it restores the run. */
 const ARCHIVE_MARKER = "archived.json";
 function isMissing(error) {
@@ -8094,11 +8281,13 @@ function trialPath(root, runId, trialId) {
 }
 function runStatus(summary) {
 	if (summary.status === "running") return "running";
+	if (summary.status === "cancelled") return "cancelled";
 	if (summary.status !== "completed") return "errored";
 	return summary.failed === 0 && summary.passed === summary.trialCount ? "passed" : "failed";
 }
 function trialStatus(summary) {
 	if (summary.status === "running") return "running";
+	if (summary.status === "cancelled") return "cancelled";
 	if (summary.status !== "completed") return "errored";
 	return summary.scoring?.passed === true ? "passed" : "failed";
 }
@@ -8143,10 +8332,15 @@ function createDashboardReportReader(root, registry) {
 		const summary = await ifPresent(() => readRunSummary(root, id));
 		const trialIds = await ifPresent(() => readdir(resolve(root, id, "trials")));
 		const trials = await Promise.all((trialIds ?? []).map((trialId) => ifPresent(() => readTrialSummary(root, id, trialId))));
+		const trialManifests = await Promise.all((trialIds ?? []).map((trialId) => ifPresent(() => readTrialManifest(root, id, trialId))));
 		const scores = trials.flatMap((trial) => trial?.scoring?.overall === void 0 ? [] : [trial.scoring.overall]);
+		const scorers = runScorers(trials);
+		const datasets = runDatasets(trialManifests);
+		const archived = await ifPresent(() => stat(resolve(root, id, ARCHIVE_MARKER)));
+		const interrupted = !summary && ownerIsGone(manifest.process);
 		return {
 			id,
-			...await ifPresent(() => stat(resolve(root, id, "archived.json"))) ? { archived: true } : {},
+			...archived ? { archived: true } : {},
 			...manifest.source ? { source: manifest.source } : {},
 			evalId: manifest.evalId,
 			...manifest.suiteId ? { suiteId: manifest.suiteId } : {},
@@ -8160,26 +8354,31 @@ function createDashboardReportReader(root, registry) {
 				manifest.aut.id,
 				manifest.aut.version
 			].filter(Boolean).join(" / ") } : {},
-			status: summary ? runStatus(summary) : "running",
+			status: summary ? runStatus(summary) : interrupted ? "cancelled" : "running",
+			...interrupted ? { interrupted: true } : {},
 			startedAt: manifest.startedAt,
 			...summary?.endedAt ? { completedAt: summary.endedAt } : {},
 			...summary?.durationMs !== void 0 ? { durationMs: summary.durationMs } : {},
 			completedTrials: summary?.trialCount ?? trials.filter(Boolean).length,
 			requestedTrials: summary?.trialCount ?? registry.get(manifest.evalId)?.policy?.trials ?? 1,
-			...scores.length ? { score: scores.reduce((sum, score) => sum + score, 0) / scores.length } : {}
+			...scores.length ? { score: scores.reduce((sum, score) => sum + score, 0) / scores.length } : {},
+			...scorers.length ? { scorers } : {},
+			...datasets.length ? { datasets } : {}
 		};
 	}
 	async function listTrials(runId) {
 		assertUuid(runId, "run ID");
 		const ids = await ifPresent(() => readdir(resolve(root, runId, "trials")));
 		if (!ids) return [];
+		const runManifest = await ifPresent(() => readRunManifest(root, runId));
+		const ownerGone = ownerIsGone(runManifest?.process);
 		return (await Promise.all(ids.map(async (id) => {
 			const manifest = await readTrialManifest(root, runId, id);
 			const summary = await ifPresent(() => readTrialSummary(root, runId, id));
 			return {
 				id,
 				index: manifest.trialIndex,
-				status: summary ? trialStatus(summary) : "running",
+				status: summary ? trialStatus(summary) : ownerGone ? "cancelled" : "running",
 				startedAt: manifest.startedAt,
 				...summary?.endedAt ? { completedAt: summary.endedAt } : {},
 				...summary?.durationMs !== void 0 ? { durationMs: summary.durationMs } : {},
@@ -8208,7 +8407,7 @@ function createDashboardReportReader(root, registry) {
 		const file = resolve(candidateRoot, relativePath);
 		if (file !== candidateRoot && !file.startsWith(`${candidateRoot}${sep}`)) throw new Error("Invalid workspace path");
 		if (!(await stat(file)).isFile()) throw new Error("Workspace path is not a file");
-		return new Response(await Bun.file(file).arrayBuffer(), { headers: { "content-type": fileContentType(file) } });
+		return new Response(await readFile(file), { headers: { "content-type": fileContentType(file) } });
 	}
 	async function trajectory(runId, trialId) {
 		trialPath(root, runId, trialId);
@@ -8241,7 +8440,7 @@ async function walkArtifacts(root) {
 			} else result.push({
 				path,
 				kind: "file",
-				size: Bun.file(resolve(directory, entry.name)).size
+				size: (await stat(resolve(directory, entry.name))).size
 			});
 		}
 	}
@@ -8254,7 +8453,7 @@ async function walkArtifacts(root) {
 const MARKER = ".evalkit-dashboard";
 const quietStep = (_label, work) => work(() => {});
 async function buildDashboard(options, step = quietStep) {
-	const { registry, reportRoot, dashboardRoot, outDir } = options;
+	const { registry, root, reportRoot, dashboardRoot, outDir } = options;
 	const template = await readFile(resolve(dashboardRoot, "index.html"), "utf8");
 	const data = resolve(outDir, "data");
 	const write = async (path, value) => {
@@ -8269,7 +8468,8 @@ async function buildDashboard(options, step = quietStep) {
 		await writeFile(resolve(outDir, "index.html"), dashboardIndex(template, { base: "./" }));
 	}, () => "read-only UI with relative assets");
 	await step("Eval catalog", async () => {
-		await write("catalog.json", { evals: registry.catalog() });
+		await write("catalog.json", { evals: registry.catalog({ root }) });
+		await write("project.json", { project: options.source ? projectInfo(options.source) : null });
 		await write("evals.json", { evals: registry.metadata() });
 		await write("suites.json", { suites: registry.suiteMetadata() });
 		return {
@@ -8277,6 +8477,21 @@ async function buildDashboard(options, step = quietStep) {
 			suites: registry.suites.length
 		};
 	}, ({ evals, suites }) => `${count(evals, "eval")} · ${count(suites, "suite")}`);
+	await step("Datasets", async () => {
+		let files = 0;
+		for (const [id, dataset] of projectDatasets(registry)) {
+			const listing = await datasetListing(dataset, root ?? process.cwd());
+			await write(`datasets/${id}/files.json`, listing);
+			if (listing.kind !== "static" || dataset.visibility === "evaluator") continue;
+			for (const entry of listing.files) {
+				const file = resolve(data, "datasets", id, "files", entry.path);
+				await mkdir(dirname(file), { recursive: true });
+				await writeFile(file, await staticDatasetFile(dataset, root ?? process.cwd(), entry.path));
+				files++;
+			}
+		}
+		return files;
+	}, (files) => count(files, "dataset file"));
 	const reports = createDashboardReportReader(reportRoot, registry);
 	const snapshot = await step("Run reports", async (progress) => {
 		const all = await reports.listRuns();
@@ -8284,13 +8499,16 @@ async function buildDashboard(options, step = quietStep) {
 		await write("runs.json", { runs });
 		let trials = 0;
 		let files = 0;
-		const trialIds = [];
+		const runTrials = [];
 		for (const [index, run] of runs.entries()) {
 			progress(`run ${index + 1}/${runs.length} · ${run.id.slice(0, 8)}`);
 			const written = await snapshotRun(reports, run.id, directorySink(resolve(data, "runs", run.id)));
 			trials += written.trialIds.length;
 			files += written.files;
-			trialIds.push(...written.trialIds);
+			runTrials.push({
+				id: run.id,
+				trials: written.trialIds
+			});
 		}
 		return {
 			runs: runs.length,
@@ -8298,16 +8516,16 @@ async function buildDashboard(options, step = quietStep) {
 			files,
 			skipped: reports.legacyRunIds().length,
 			archived: all.length - runs.length,
-			runIds: runs.map((run) => run.id),
-			trialIds
+			runTrials
 		};
 	}, ({ runs, trials, files, skipped, archived }) => `${count(runs, "run")} · ${count(trials, "trial")} · ${count(files, "candidate file")}` + (archived ? ` · ${archived} archived excluded` : "") + (skipped ? ` · ${skipped} older-format skipped` : ""));
 	await step("Route pages", async () => {
+		const catalog = registry.catalog();
 		const routes = staticRoutes({
-			suites: registry.suites.map(authoringId),
-			evals: registry.catalog().map((entry) => entry.path),
-			runs: snapshot.runIds,
-			trials: snapshot.trialIds
+			scenarios: catalog.map((entry) => entry.id),
+			agents: catalog.flatMap((entry) => entry.agent.id ? [entry.agent.id] : []),
+			datasets: catalog.flatMap((entry) => entry.datasets.map((dataset) => dataset.id)),
+			runs: snapshot.runTrials
 		});
 		for (const route of routes) {
 			const file = resolve(outDir, `${route}.html`);
@@ -8317,7 +8535,7 @@ async function buildDashboard(options, step = quietStep) {
 		await writeFile(resolve(outDir, "404.html"), dashboardIndex(template, { base: "/" }));
 		return routes.length;
 	}, (pages) => `${count(pages, "page")} with clean /path URLs`);
-	const { runIds: _runs, trialIds: _trials, ...summary } = snapshot;
+	const { runTrials: _runs, ...summary } = snapshot;
 	return {
 		outDir,
 		...summary
@@ -8330,18 +8548,21 @@ async function buildDashboard(options, step = quietStep) {
 * skipped; those pages still open by navigating inside the app.
 */
 function staticRoutes(ids) {
-	const safe = (id) => id.length > 0 && !/[/\\\0]/.test(id) && id !== "." && id !== "..";
-	const under = (prefix, values) => values.filter(safe).map((id) => `${prefix}/${id}`);
+	const safe = (id) => id.length > 0 && !/[/\\\0#?%]/.test(id) && id !== "." && id !== "..";
+	const under = (prefix, values = []) => [...new Set(values)].filter(safe).map((id) => `${prefix}/${id}`);
 	return [
-		"suites",
-		"agents",
-		"fixtures",
+		"scoreboard",
 		"runs",
-		...under("suites", ids.suites),
-		...under("evals", ids.evals),
-		...under("runs", ids.runs),
-		...under("trials", ids.trials),
-		...under("workspaces", ids.trials)
+		"trials",
+		"workspaces",
+		"library/agents",
+		"library/judges",
+		"library/scenarios",
+		"library/datasets",
+		...under("library/scenarios", ids.scenarios),
+		...under("library/agents", ids.agents),
+		...under("library/datasets", ids.datasets),
+		...ids.runs.filter((run) => safe(run.id)).flatMap((run) => [`runs/${run.id}`, ...run.trials.filter(safe).flatMap((trial) => [`runs/${run.id}/trials/${trial}`, `runs/${run.id}/trials/${trial}/workspace`])])
 	];
 }
 /** `runs/abc.html` is served at `<mount>/runs/abc`; its base is one level up. */
@@ -8516,6 +8737,124 @@ function selectDashboardCell(matrix, evalId, parameters) {
 	return selection;
 }
 //#endregion
+//#region ../cli/src/runtime.ts
+/**
+* The few platform services the CLI needs, on Bun or Node. Everything else
+* uses `node:` APIs, which both runtimes provide.
+*/
+const onBun = typeof globalThis.Bun !== "undefined";
+/** Node 22.18 is the first 22.x that runs the CLI's own code unflagged. */
+function assertSupportedRuntime(version = process.versions.node) {
+	if (onBun) return;
+	const [major = 0, minor = 0] = version.split(".").map(Number);
+	if (major < 22 || major === 22 && minor < 18) throw new Error(`EvalKit needs Node 22.18 or later, or Bun; this is Node ${version}.`);
+}
+/** Serves a fetch handler on `port` (0 picks a free one). */
+async function serve(fetch, port) {
+	if (onBun) {
+		const server = globalThis.Bun.serve({
+			port,
+			fetch
+		});
+		return {
+			port: server.port,
+			stop: async () => server.stop(true)
+		};
+	}
+	const server = createServer((incoming, outgoing) => {
+		(async () => {
+			try {
+				const response = await fetch(toRequest(incoming, port));
+				outgoing.writeHead(response.status, [...response.headers]);
+				if (response.body) for await (const chunk of response.body) outgoing.write(chunk);
+				outgoing.end();
+			} catch (error) {
+				outgoing.statusCode = 500;
+				outgoing.end(error instanceof Error ? error.message : "Internal error");
+			}
+		})();
+	});
+	await new Promise((resolvePromise, reject) => {
+		server.once("error", reject);
+		server.listen(port, () => resolvePromise());
+	});
+	const address = server.address();
+	return {
+		port: typeof address === "object" && address ? address.port : port,
+		stop: () => new Promise((resolvePromise) => server.close(() => resolvePromise()))
+	};
+}
+function toRequest(incoming, port) {
+	const host = incoming.headers.host ?? `localhost:${port}`;
+	const url = new URL(incoming.url ?? "/", `http://${host}`);
+	const headers = new Headers();
+	for (const [name, value] of Object.entries(incoming.headers)) if (value !== void 0) for (const item of Array.isArray(value) ? value : [value]) headers.append(name, item);
+	const method = incoming.method ?? "GET";
+	return new Request(url, {
+		method,
+		headers,
+		...method === "GET" || method === "HEAD" ? {} : {
+			body: incoming,
+			duplex: "half"
+		}
+	});
+}
+const sleep = (ms) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
+/** Best effort; the printed URL stays authoritative. */
+function openInBrowser(url) {
+	const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
+	try {
+		const child = spawn(opener, [url], {
+			stdio: "ignore",
+			detached: true
+		});
+		child.on("error", () => {});
+		child.unref();
+	} catch {}
+}
+let jiti;
+/**
+* Imports a project's TypeScript or JavaScript module. Bun loads TypeScript
+* natively; on Node, jiti handles TypeScript, including `.js` specifiers that
+* point at `.ts` files.
+*/
+async function importProjectModule(file) {
+	if (onBun || /\.(m?js|cjs)$/.test(file)) return import(pathToFileURL(file).href);
+	if (!jiti) {
+		const { createJiti } = await import("jiti");
+		jiti = createJiti(import.meta.url, { moduleCache: false });
+	}
+	return jiti.import(file);
+}
+/**
+* A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
+* spans segments, and `{a,b}` is an alternation.
+*/
+function globMatcher(pattern) {
+	let source = "";
+	for (let index = 0; index < pattern.length; index++) {
+		const char = pattern[index];
+		if (char === "*" && pattern[index + 1] === "*") {
+			index++;
+			if (pattern[index + 1] === "/") {
+				index++;
+				source += "(?:.*/)?";
+			} else source += ".*";
+		} else if (char === "*") source += "[^/]*";
+		else if (char === "?") source += "[^/]";
+		else if (char === "{") {
+			const end = pattern.indexOf("}", index);
+			if (end === -1) source += "\\{";
+			else {
+				source += `(?:${pattern.slice(index + 1, end).split(",").map((part) => part.replace(/[.+^$()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*")).join("|")})`;
+				index = end;
+			}
+		} else source += char.replace(/[.+^$()|[\]\\]/g, "\\$&");
+	}
+	const expression = new RegExp(`^${source}$`);
+	return (path) => expression.test(path);
+}
+//#endregion
 //#region ../cli/src/project.ts
 const defaultInclude = ["**/*.eval.ts", "**/*.eval.js"];
 const defaultExclude = [
@@ -8538,10 +8877,16 @@ function evaluation(value, source) {
 	if (typeof candidate.agent?.start !== "function" || !Array.isArray(candidate.transcript) || !Array.isArray(candidate.scoring)) throw new Error(`Invalid eval exported by ${source}`);
 	return candidate;
 }
+/** The suite a discovered file belongs to: the first directory below the discovery root. */
+function directorySuite(relativePath) {
+	const parts = relativePath.split("/");
+	return parts.length > 1 ? parts[0] : void 0;
+}
 async function discover(root, config) {
-	const directory = resolve(root, config.testDir ?? "evals");
-	const includes = (config.include ?? defaultInclude).map((pattern) => new Bun.Glob(pattern));
-	const excludes = [...defaultExclude, ...config.exclude ?? []].map((pattern) => new Bun.Glob(pattern));
+	const testDir = config.testDir ?? "scenarios";
+	const directory = resolve(root, testDir);
+	const includes = (config.include ?? defaultInclude).map((pattern) => ({ match: globMatcher(pattern) }));
+	const excludes = [...defaultExclude, ...config.exclude ?? []].map((pattern) => ({ match: globMatcher(pattern) }));
 	const paths = [];
 	async function walk(dir) {
 		const entries = await readdir(dir, { withFileTypes: true });
@@ -8557,16 +8902,42 @@ async function discover(root, config) {
 	try {
 		await walk(directory);
 	} catch (error) {
-		if (error.code === "ENOENT") throw new Error(`Eval directory not found: ${directory}`);
-		throw error;
+		if (error.code !== "ENOENT") throw error;
+		const hint = config.testDir === void 0 && await isDirectory(resolve(root, "evals")) ? ` Scenarios now live in scenarios/: move evals/ there, or set testDir: 'evals' in evalkit.config.ts.` : "";
+		throw new Error(`Eval directory not found: ${directory}.${hint}`);
 	}
-	const evals = [];
+	const directorySuites = config.discovery?.suites ?? true;
+	const standalone = [];
+	const bySuite = /* @__PURE__ */ new Map();
+	const loadErrors = [];
 	for (const file of paths.sort()) {
-		const module = await import(pathToFileURL(file).href);
-		for (const value of Array.isArray(module.default) ? module.default : [module.default]) evals.push(evaluation(value, file));
+		const fromRoot = relative(root, file).split(sep).join("/");
+		const suiteId = directorySuites ? directorySuite(relative(directory, file).split(sep).join("/")) : void 0;
+		try {
+			if (suiteId !== void 0 && !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(suiteId)) throw new Error(`Directory ${testDir}/${suiteId} names a suite, so it must be lowercase kebab-case (or set discovery.suites: false)`);
+			const exported = await (await importProjectModule(file)).default;
+			const evals = (Array.isArray(exported) ? exported : [exported]).map((value) => evaluation(value, fromRoot));
+			if (suiteId === void 0) standalone.push(...evals);
+			else bySuite.set(suiteId, [...bySuite.get(suiteId) ?? [], ...evals]);
+		} catch (error) {
+			loadErrors.push({
+				file: fromRoot,
+				...suiteId === void 0 ? {} : { suiteId },
+				error: recordError(error)
+			});
+		}
 	}
-	if (!evals.length) throw new Error(`No eval files found in ${directory}`);
-	return evals;
+	if (!paths.length) throw new Error(`No eval files found in ${directory}`);
+	return {
+		registrations: [...standalone, ...[...bySuite].map(([id, evals]) => defineSuite({
+			id,
+			evals
+		}))],
+		loadErrors
+	};
+}
+async function isDirectory(path) {
+	return stat(path).then((info) => info.isDirectory(), () => false);
 }
 async function loadProject(cwd, configPath) {
 	const candidates = configPath ? [resolve(cwd, configPath)] : [
@@ -8580,12 +8951,18 @@ async function loadProject(cwd, configPath) {
 	if (present.length > 1) throw new Error("Multiple EvalKit configs found; select one with --config");
 	const file = present[0];
 	const root = file ? dirname(file) : resolve(cwd);
-	const config = file ? (await import(pathToFileURL(file).href)).default : {};
+	const config = file ? (await importProjectModule(file)).default : {};
 	if (!config || typeof config !== "object" || Array.isArray(config)) throw new Error("Config must default-export defineConfig({...})");
 	for (const [key, value] of Object.entries(config.execution ?? {})) if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) throw new Error(`execution.${key} must be a positive integer`);
 	let registry = config.registry;
-	if (!file && !configPath && await exists$1(resolve(root, "src/registry.ts"))) registry = (await import(pathToFileURL(resolve(root, "src/registry.ts")).href)).default;
-	if (!registry) registry = registerEvals(config.evals ?? await discover(root, config));
+	if (!file && !configPath && await exists$1(resolve(root, "src/registry.ts"))) registry = (await importProjectModule(resolve(root, "src/registry.ts"))).default;
+	let loadErrors = [];
+	if (!registry && config.evals) registry = registerEvals(config.evals);
+	if (!registry) {
+		const discovered = await discover(root, config);
+		loadErrors = discovered.loadErrors;
+		registry = registerEvals(discovered.registrations);
+	}
 	if (!registry?.evals || !registry.get) throw new Error("Invalid explicit eval registry");
 	const ids = /* @__PURE__ */ new Set();
 	for (const e of registry.evals) {
@@ -8608,7 +8985,8 @@ async function loadProject(cwd, configPath) {
 	return {
 		root,
 		config,
-		registry
+		registry,
+		loadErrors
 	};
 }
 //#endregion
@@ -8717,125 +9095,206 @@ async function confirm(question) {
 	}
 }
 //#endregion
-//#region ../cli/src/source.ts
-const exec = promisify(execFile);
-/**
-* Normalize a git remote to a stable repository key: lowercase host, no
-* credentials, port, scheme, `.git`, or trailing slash.
-*
-*   git@GitLab.example.com:group/sub/repo.git → gitlab.example.com/group/sub/repo
-*   https://user:token@github.com/o/r/        → github.com/o/r
-*   /srv/repos/thing.git (no host)            → local/thing
-*/
-function normalizeRepoUrl(url) {
-	const trimmed = url.trim().replace(/\/+$/, "").replace(/\.git$/, "");
-	const scp = /^(?:[^@/]+@)?([^:/]+):(?!\/)(.+)$/.exec(trimmed);
-	if (scp && !/^[a-z]+:\/\//i.test(trimmed)) return `${scp[1].toLowerCase()}/${scp[2].replace(/^\/+/, "")}`;
-	try {
-		const parsed = new URL(trimmed);
-		if (parsed.protocol !== "file:" && parsed.hostname) return `${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/+$/, "")}`;
-	} catch {}
-	return `local/${basename(trimmed)}`;
-}
-async function git(cwd, ...args) {
-	try {
-		const { stdout } = await exec("git", [
-			"-C",
-			cwd,
-			...args
-		], { timeout: 1e4 });
-		return stdout.trim() || void 0;
-	} catch {
-		return;
+//#region ../cli/src/run-reporter.ts
+function scenarioOutcome(result) {
+	if (result.status === "cancelled") return "cancelled";
+	return result.status === "completed" && (result.trials ? result.trials.length > 0 && result.trials.every((trial) => trial.status === "completed" && trial.scoring?.passed) : Boolean(result.scoring?.passed)) ? "passed" : "failed";
+}
+function passAtFor(definition, name) {
+	const rule = [...definition.transcript, ...definition.scoring].find((step) => step.kind === "predicate" && step.name === name);
+	return rule?.kind === "predicate" ? rule.passAt : void 0;
+}
+function value(score) {
+	return score.value === void 0 ? "" : score.value.toFixed(1);
+}
+function scoreReason(definition, score) {
+	if (score.error) return `${score.name} error: ${score.error.message}`;
+	const passAt = passAtFor(definition, score.name);
+	return `${score.name} ${value(score)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+}
+function checkpointReason(definition, checkpoint) {
+	if (checkpoint.status === "error") return `step ${checkpoint.step} ${checkpoint.name} error: ${checkpoint.error?.message ?? ""}`;
+	const passAt = passAtFor(definition, checkpoint.name);
+	return `step ${checkpoint.step} ${checkpoint.name} ${checkpoint.value === void 0 ? checkpoint.status : value(checkpoint)}${passAt === void 0 ? "" : ` (passes at ${passAt})`}`;
+}
+/** What went wrong in a scenario, deduplicated across its trials. */
+function reasonsFor(definition, trials) {
+	const reasons = /* @__PURE__ */ new Set();
+	const skipped = /* @__PURE__ */ new Set();
+	for (const trial of trials) {
+		if (trial.status === "failed" && trial.error) reasons.add(`error: ${trial.error.message}`);
+		for (const checkpoint of trial.scoring?.checkpoints ?? []) if (checkpoint.status === "failed" || checkpoint.status === "error") reasons.add(checkpointReason(definition, checkpoint));
+		for (const score of trial.scoring?.results ?? []) if (score.skipped) skipped.add(score.name);
+		else if (!score.passed) reasons.add(scoreReason(definition, score));
+		for (const name of trial.scoring?.skippedScorers ?? []) if (trial.status !== "cancelled") reasons.add(`${name} not run`);
 	}
-}
-const posix = (path) => path.split(sep).join("/");
-const projectPath = (root, projectRoot) => root ? posix(relative(root, projectRoot)) : "";
-function ciSource(env, projectRoot) {
-	if (env.EVALKIT_SOURCE_REPO && env.EVALKIT_SOURCE_COMMIT) return {
-		repo: normalizeRepoUrl(env.EVALKIT_SOURCE_REPO),
-		project: env.EVALKIT_SOURCE_PROJECT ?? "",
-		commit: env.EVALKIT_SOURCE_COMMIT,
-		...env.EVALKIT_SOURCE_REF ? { ref: env.EVALKIT_SOURCE_REF } : {},
-		...env.EVALKIT_SOURCE_COMMITTED_AT ? { committedAt: env.EVALKIT_SOURCE_COMMITTED_AT } : {}
-	};
-	if (env.GITLAB_CI === "true" && env.CI_PROJECT_URL && env.CI_COMMIT_SHA) return {
-		repo: normalizeRepoUrl(env.CI_PROJECT_URL),
-		project: projectPath(env.CI_PROJECT_DIR, projectRoot),
-		commit: env.CI_COMMIT_SHA,
-		...env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME ? { ref: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ?? env.CI_COMMIT_REF_NAME } : {},
-		...env.CI_COMMIT_TIMESTAMP ? { committedAt: env.CI_COMMIT_TIMESTAMP } : {},
-		ci: {
-			provider: "gitlab",
-			...env.CI_PIPELINE_ID ? { pipeline: env.CI_PIPELINE_ID } : {},
-			...env.CI_JOB_ID ? { job: env.CI_JOB_ID } : {},
-			...env.CI_JOB_URL ? { url: env.CI_JOB_URL } : {}
-		}
+	return {
+		reasons: [...reasons],
+		skipped: [...skipped]
 	};
-	if (env.GITHUB_ACTIONS === "true" && env.GITHUB_REPOSITORY && env.GITHUB_SHA) {
-		const server = env.GITHUB_SERVER_URL ?? "https://github.com";
-		return {
-			repo: normalizeRepoUrl(`${server}/${env.GITHUB_REPOSITORY}`),
-			project: projectPath(env.GITHUB_WORKSPACE, projectRoot),
-			commit: env.GITHUB_SHA,
-			...env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME ? { ref: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME } : {},
-			ci: {
-				provider: "github",
-				...env.GITHUB_RUN_ID ? { pipeline: env.GITHUB_RUN_ID } : {},
-				...env.GITHUB_JOB ? { job: env.GITHUB_JOB } : {},
-				...env.GITHUB_RUN_ID ? { url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}` } : {}
-			}
-		};
-	}
 }
-/**
-* Determine which repository and commit this project's eval code comes from.
-* Order: EVALKIT_SOURCE_* variables, GitLab/GitHub CI variables, then git.
-* Returns undefined outside git with no CI variables ("untracked" results).
-*
-* `outputDirs` (results, sandbox, snapshot) never make the tree dirty, even
-* when a project forgot to ignore them.
-*/
-async function detectSource(givenRoot, options = {}) {
-	const env = options.env ?? process.env;
-	const projectRoot = await realpath(givenRoot).catch(() => givenRoot);
-	const top = await git(projectRoot, "rev-parse", "--show-toplevel");
-	const excludes = top ? (options.outputDirs ?? []).map((dir) => join(projectRoot, relative(givenRoot, dir))).map((dir) => posix(relative(top, dir))).filter((dir) => dir && !dir.startsWith("..")).map((dir) => `:(exclude)${dir}`) : [];
-	const dirty = top ? Boolean(await git(top, "status", "--porcelain", "--", ".", ...excludes)) : false;
-	const fromCi = ciSource(env, projectRoot);
-	if (fromCi) return {
-		...fromCi,
-		...top && !env.EVALKIT_SOURCE_PROJECT ? { project: projectPath(top, projectRoot) } : {},
-		dirty
-	};
-	if (!top) return void 0;
-	const commit = await git(top, "rev-parse", "HEAD");
-	if (!commit) return void 0;
-	const remote = await git(top, "config", "--get", "remote.origin.url") ?? (await git(top, "remote"))?.split("\n").map((name) => name.trim()).find(Boolean);
-	const remoteUrl = remote && !remote.includes("/") && !remote.includes(":") ? await git(top, "config", "--get", `remote.${remote}.url`) : remote;
-	const ref = await git(top, "symbolic-ref", "--short", "-q", "HEAD");
-	const committedAt = await git(top, "show", "-s", "--format=%cI", "HEAD");
+function scenarioResult(cell, result, suiteId) {
+	const evalId = authoringId(cell.eval);
+	const trials = result.trials ?? [result];
 	return {
-		repo: normalizeRepoUrl(remoteUrl ?? top),
-		project: projectPath(top, projectRoot),
-		commit,
-		...ref ? { ref } : {},
-		...committedAt ? { committedAt } : {},
-		dirty
+		label: suiteId ? `${suiteId}/${evalId}` : evalId,
+		evalId,
+		...suiteId ? { suiteId } : {},
+		parameters: cell.parameters,
+		outcome: scenarioOutcome(result),
+		...result.durationMs !== void 0 ? { durationMs: result.durationMs } : {},
+		result,
+		...reasonsFor(cell.eval, trials)
 	};
 }
-/** The project's evals and suites, saved with each run (`catalog.json`). */
-function runCatalog(registry) {
+function parameterLabel(parameters) {
+	return Object.entries(parameters).map(([key, item]) => `${key}=${typeof item === "string" ? item : JSON.stringify(item)}`).join(" ");
+}
+/** One printed line for a finished scenario; failing reasons on the same line. */
+function scenarioLine(scenario) {
+	const mark = scenario.outcome === "passed" ? green("✓") : scenario.outcome === "cancelled" ? yellow("◌") : red("✗");
+	const parameters = parameterLabel(scenario.parameters);
+	const trials = scenario.result.trials;
+	const trialCount = trials && trials.length > 1 ? dim(` ${trials.filter((trial) => trial.status === "completed" && trial.scoring?.passed).length}/${trials.length} trials`) : "";
+	const details = [
+		...scenario.outcome === "failed" ? scenario.reasons : [],
+		...scenario.outcome === "cancelled" ? ["cancelled"] : [],
+		...scenario.skipped.map((name) => `${name} skipped`)
+	];
+	return `  ${mark} ${bold(scenario.label)}${parameters ? ` ${dim(parameters)}` : ""}${trialCount}${details.length ? `   ${dim(details.join(" · "))}` : ""}`;
+}
+/**
+* Prints run progress. On a terminal, failures print as they happen and a live
+* line shows counts and what is running. Elsewhere (CI), every finished
+* scenario prints one plain line.
+*/
+function createRunReporter(options) {
+	const write = options.write ?? ((text) => process.stdout.write(text));
+	const now = options.now ?? (() => performance.now());
+	const running = /* @__PURE__ */ new Map();
+	const scenarios = [];
+	let liveShown = false;
+	let timer;
+	function clearLive() {
+		if (liveShown) write("\r\x1B[2K");
+		liveShown = false;
+	}
+	function liveLine() {
+		const failed = scenarios.filter((s) => s.outcome === "failed").length;
+		const active = [...running.values()].slice(0, 2).map((activity) => {
+			const seconds = Math.round((now() - activity.startedAt) / 1e3);
+			return `${activity.label} (${activity.doing ? `${activity.doing}, ` : ""}${seconds}s)`;
+		});
+		const more = running.size > 2 ? ", …" : "";
+		return `  ${cyan("◇")} ${scenarios.length}/${options.total}${failed ? ` · ${red(`${failed} failed`)}` : ""}${active.length ? dim(` · running: ${active.join(", ")}${more}`) : ""}`;
+	}
+	function drawLive() {
+		if (!options.interactive) return;
+		const line = liveLine();
+		const width = process.stdout.columns ?? 120;
+		write(`\r\x1b[2K${line.length > width + 40 ? line.slice(0, width + 40) : line}`);
+		liveShown = true;
+	}
+	function print(line) {
+		clearLive();
+		write(`${line}\n`);
+		drawLive();
+	}
 	return {
-		catalog: registry.catalog(),
-		evals: registry.metadata(),
-		suites: registry.suiteMetadata()
+		start() {
+			if (options.interactive) {
+				timer = setInterval(drawLive, 100);
+				timer.unref?.();
+			}
+		},
+		onEvent(trial, event) {
+			if (event.source === "runner" && event.kind === "trial-started") {
+				running.set(trial.trialId, {
+					label: options.labelFor(trial),
+					startedAt: now()
+				});
+				return;
+			}
+			const activity = running.get(trial.trialId);
+			if (!activity) return;
+			if (event.source === "runner" && (event.kind === "trial-completed" || event.kind === "error")) running.delete(trial.trialId);
+			else if (event.source === "aut" && event.kind === "tool-call") activity.doing = `tool ${event.name}`;
+			else if (event.source === "runner" && event.kind === "scorer-started") activity.doing = `scoring ${event.scorer}`;
+			else if (event.source === "runner" && event.kind === "checkpoint-started") activity.doing = `check ${event.name}`;
+		},
+		onScenario(scenario) {
+			scenarios.push(scenario);
+			for (const trial of scenario.result.trials ?? [scenario.result]) running.delete(trial.trialId);
+			if (!options.interactive || options.verbose || scenario.outcome !== "passed") print(scenarioLine(scenario));
+		},
+		/** Called on the first Ctrl+C. */
+		cancelling() {
+			print(`\n  ${yellow("◌")} Cancelling: finishing running trials. Press Ctrl+C again to exit now.`);
+		},
+		finish(durationMs) {
+			if (timer) clearInterval(timer);
+			clearLive();
+			return {
+				passed: scenarios.filter((s) => s.outcome === "passed").length,
+				failed: scenarios.filter((s) => s.outcome === "failed").length,
+				cancelled: scenarios.filter((s) => s.outcome === "cancelled").length,
+				scenarios: scenarios.length,
+				durationMs
+			};
+		},
+		scenarios: () => scenarios
 	};
 }
-/** `abc1234` or `abc1234+dirty`. */
-function shortSource(source) {
-	return `${source.commit.slice(0, 7)}${source.dirty ? "+dirty" : ""}`;
+function totalsLine(totals) {
+	const parts = [
+		green(`${totals.passed} passed`),
+		totals.failed ? red(`${totals.failed} failed`) : "",
+		totals.cancelled ? yellow(`${totals.cancelled} cancelled`) : ""
+	].filter(Boolean);
+	return `  ${bold(parts.join(" · "))} ${dim(`· ${plural(totals.scenarios, "scenario")} in ${duration(totals.durationMs)}`)}`;
+}
+//#endregion
+//#region ../cli/src/junit.ts
+function escape(text) {
+	return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g, "");
+}
+const seconds = (ms) => ((ms ?? 0) / 1e3).toFixed(3);
+/**
+* A JUnit XML report: one test suite per EvalKit suite (`default` for
+* scenarios without one) and one test case per scenario, per matrix cell
+* when it has parameters.
+*/
+function junitReport(scenarios) {
+	const bySuite = /* @__PURE__ */ new Map();
+	for (const scenario of scenarios) {
+		const suite = scenario.suiteId ?? "default";
+		bySuite.set(suite, [...bySuite.get(suite) ?? [], scenario]);
+	}
+	const suites = [...bySuite].map(([suite, cases]) => {
+		const failures = cases.filter((c) => c.outcome === "failed").length;
+		const skipped = cases.filter((c) => c.outcome === "cancelled" || c.outcome === "passed" && c.skipped.length > 0 && allSkipped(c)).length;
+		const time = cases.reduce((sum, c) => sum + (c.durationMs ?? 0), 0);
+		const testcases = cases.map((scenario) => {
+			const parameters = parameterLabel(scenario.parameters);
+			const attributes = `name="${escape(parameters ? `${scenario.evalId} [${parameters}]` : scenario.evalId)}" classname="${escape(suite)}" time="${seconds(scenario.durationMs)}"`;
+			if (scenario.outcome === "failed") return `    <testcase ${attributes}>\n      <failure message="${escape(scenario.reasons.join("; ") || "failed")}">${escape(scenario.reasons.join("\n"))}</failure>\n    </testcase>`;
+			if (scenario.outcome === "cancelled") return `    <testcase ${attributes}>\n      <skipped message="cancelled"/>\n    </testcase>`;
+			if (allSkipped(scenario)) return `    <testcase ${attributes}>\n      <skipped message="${escape(`all scorers skipped: ${scenario.skipped.join(", ")}`)}"/>\n    </testcase>`;
+			return `    <testcase ${attributes}/>`;
+		});
+		return `  <testsuite name="${escape(suite)}" tests="${cases.length}" failures="${failures}" errors="0" skipped="${skipped}" time="${seconds(time)}">\n${testcases.join("\n")}\n  </testsuite>`;
+	});
+	return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="evalkit" tests="${scenarios.length}" failures="${scenarios.filter((c) => c.outcome === "failed").length}">\n${suites.join("\n")}\n</testsuites>\n`;
 }
+/** True when every score of every trial was skip()ped. */
+function allSkipped(scenario) {
+	const results = (scenario.result.trials ?? [scenario.result]).flatMap((trial) => trial.scoring?.results ?? []);
+	return results.length > 0 && results.every((score) => score.skipped);
+}
+//#endregion
+//#region ../cli/src/version.ts
+/** The CLI version; kept in sync with package.json when releasing. */
+const VERSION = "0.6.0";
 //#endregion
 //#region ../cli/src/run-command.ts
 function parseRunArgs(args) {
@@ -8875,7 +9334,10 @@ function parseRunArgs(args) {
 			json: { type: "boolean" },
 			"dry-run": { type: "boolean" },
 			all: { type: "boolean" },
-			local: { type: "boolean" }
+			local: { type: "boolean" },
+			junit: { type: "string" },
+			verbose: { type: "boolean" },
+			"skip-broken": { type: "boolean" }
 		}
 	});
 	return normalizeRunOptions(values, positionals);
@@ -8928,21 +9390,37 @@ function normalizeRunOptions(values, positionals) {
 		trials: integer("trials")
 	};
 }
+/** Load errors that affect what was asked for, so the run must not pretend it covered them. */
+function brokenSelection(loadErrors, request) {
+	if (!loadErrors.length) return [];
+	if (request.suiteId) return loadErrors.filter((error) => error.suiteId === request.suiteId);
+	if (request.evalIds.length) return request.evalIds.some((id) => !request.registry.get(id)) ? loadErrors : [];
+	return loadErrors;
+}
+/** Load errors are warnings; `print` is stderr when stdout must stay machine-readable. */
+function printLoadErrors(loadErrors, print = console.log) {
+	for (const { file, error } of loadErrors) print(`  ${yellow("✗")} ${bold(file)} failed to load:\n    ${dim(error.message)}`);
+	if (loadErrors.length) print("");
+}
 async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArgs(args)) {
 	const { values, positionals, parameters } = parsed;
+	const started = performance.now();
 	const project = await loadProject(cwd, values.config);
 	process.chdir(project.root);
 	const registry = project.registry;
 	const requestedEvals = values.eval?.flatMap((value) => value.split(",")) ?? [];
 	let matrix;
 	let suiteId;
+	const quiet = Boolean(values.json || values["dry-run"]);
+	if (!quiet) banner("run", VERSION);
 	if (values.suite) {
 		const suite = registry.suites.find((item) => item.id === values.suite);
-		if (!suite) throw new Error(`Unknown suite: ${values.suite}`);
-		suiteId = authoringId(suite);
+		const broken = project.loadErrors.filter((error) => error.suiteId === values.suite);
+		if (!suite && !broken.length) throw new Error(`Unknown suite: ${values.suite}`);
+		suiteId = values.suite;
 		matrix = defineEvalMatrix({
 			id: "default",
-			evals: suite.evals,
+			evals: suite?.evals ?? [],
 			parameters: project.config.matrix?.parameters ?? {},
 			defaults: project.config.matrix?.defaults,
 			cases: project.config.matrix?.cases,
@@ -8959,6 +9437,19 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
 			parameters: {}
 		});
 	}
+	const broken = brokenSelection(project.loadErrors, {
+		...suiteId ? { suiteId } : {},
+		evalIds: requestedEvals,
+		registry
+	});
+	printLoadErrors(project.loadErrors, quiet ? console.error : console.log);
+	if (broken.length && !values["skip-broken"] && !values["dry-run"]) throw new Error(`${plural(broken.length, "eval file")} in this selection failed to load: ${broken.map((error) => error.file).join(", ")}. Fix ${broken.length === 1 ? "it" : "them"}, or pass --skip-broken to run the rest.`);
+	const knownEvals = requestedEvals.filter((id) => registry.get(id));
+	if (values["skip-broken"] && knownEvals.length < requestedEvals.length) {
+		requestedEvals.length = 0;
+		requestedEvals.push(...knownEvals);
+		if (!requestedEvals.length) throw new Error("Every selected eval is in a file that failed to load.");
+	}
 	const selection = {
 		parameters: {},
 		overrides: { ...parameters },
@@ -8982,7 +9473,8 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
 		console.log(JSON.stringify(plan, null, 2));
 		return;
 	}
-	if (cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
+	const sweeps = Object.keys(matrix.parameters).length > 0 || Boolean(matrix.cases?.length);
+	if (sweeps && cells > (project.config.execution?.maxCells ?? 100) && !values.all) throw new Error(`Selected ${cells} cells. Narrow the selection, inspect --dry-run, or pass --all.`);
 	const reportDir = resolve(project.root, project.config.reportDir ?? "_evalkit-results");
 	const sandboxDir = resolve(project.root, project.config.sandboxDir ?? "_evalkit-sandbox");
 	const source = await detectSource(project.root, { outputDirs: [
@@ -8990,36 +9482,75 @@ async function runProjectCommand(args, cwd = process.cwd(), parsed = parseRunArg
 		sandboxDir,
 		resolve(project.root, "_evalkit-dashboard")
 	] });
-	if (!values.json) console.log(`Running ${cells} cell(s) from ${plan.matrix}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)"));
-	if ((await Effect.runPromise(runnerLogs(runMatrix(matrix, {
-		selection,
-		concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
-		trials,
-		...suiteId ? { suiteId } : {},
-		...values.local ? { runtime: "local" } : {},
-		...source ? { source } : {},
-		catalog: runCatalog(registry),
-		report: localReportStore(reportDir),
-		workspaceRoot: sandboxDir,
-		onResult: (cell, result) => {
-			if (values.json) console.log(JSON.stringify({
-				cell: {
-					key: cell.key,
-					eval: authoringId(cell.eval),
-					parameters: cell.parameters
-				},
-				result
-			}));
-			else {
-				const passed = result.status === "completed" && (result.aggregateScoring?.passRate === 1 || result.scoring?.passed);
-				console.log(`${passed ? "PASS" : "FAIL"} ${cell.eval.name ?? authoringId(cell.eval)} ${JSON.stringify(cell.parameters)}\n  report: ${result.reportLocation}`);
-				for (const trial of result.trials ?? [result]) {
-					for (const checkpoint of trial.scoring?.checkpoints ?? []) console.log(`  step ${checkpoint.step} ${checkpoint.kind} ${checkpoint.name}: ${checkpoint.status}${checkpoint.value === void 0 ? "" : ` (${checkpoint.value})`}`);
-					for (const score of trial.scoring?.results ?? []) console.log(`  ${score.kind} ${score.name}: ${score.value ?? "error"} ${score.explanation ?? ""}`);
-				}
+	const suiteOf = new Map(registry.suites.flatMap((suite) => suite.evals.map((evaluation) => [authoringId(evaluation), suite.id])));
+	const suiteFor = (evalId) => suiteId ?? suiteOf.get(evalId);
+	if (!values.json) console.log(`  Running ${plural(cells, "scenario")}${sweeps ? ` from matrix ${plan.matrix}` : ""}${suiteId ? ` from suite ${suiteId}` : ""}` + (source ? ` at ${shortSource(source)}${source.ref ? ` (${source.ref})` : ""}` : " (untracked: not in a git repository)") + "\n");
+	const reporter = createRunReporter({
+		interactive: interactive && !values.json,
+		verbose: Boolean(values.verbose),
+		total: cells,
+		labelFor: (trial) => {
+			const suite = suiteFor(trial.evalId);
+			return suite ? `${suite}/${trial.evalId}` : trial.evalId;
+		},
+		write: values.json ? () => {} : void 0
+	});
+	const controller = new AbortController();
+	const onInterrupt = () => {
+		if (controller.signal.aborted) process.exit(130);
+		controller.abort();
+		if (!values.json) reporter.cancelling();
+	};
+	process.on("SIGINT", onInterrupt);
+	reporter.start();
+	let summary;
+	try {
+		summary = await Effect.runPromise(runnerLogs(runMatrix(matrix, {
+			selection,
+			concurrency: parsed.concurrency ?? project.config.execution?.concurrency ?? 4,
+			trials,
+			...suiteId ? { suiteId } : {},
+			suiteFor: (cell) => suiteFor(authoringId(cell.eval)),
+			...values.local ? { runtime: "local" } : {},
+			...source ? { source } : {},
+			catalog: runCatalog(registry, project.root),
+			report: localReportStore(reportDir),
+			workspaceRoot: sandboxDir,
+			signal: controller.signal,
+			onEvent: (trial, event) => reporter.onEvent(trial, event),
+			onResult: (cell, result) => {
+				const scenario = scenarioResult(cell, result, suiteFor(authoringId(cell.eval)));
+				if (values.json) console.log(JSON.stringify({
+					cell: {
+						key: cell.key,
+						eval: authoringId(cell.eval),
+						parameters: cell.parameters
+					},
+					outcome: scenario.outcome,
+					result
+				}));
+				reporter.onScenario(scenario);
 			}
-		}
-	})))).failed) process.exitCode = 1;
+		})));
+	} finally {
+		process.off("SIGINT", onInterrupt);
+	}
+	const totals = reporter.finish(performance.now() - started);
+	if (values.junit) {
+		const file = resolve(cwd, values.junit);
+		await mkdir(dirname(file), { recursive: true });
+		await writeFile(file, junitReport(reporter.scenarios()));
+	}
+	if (values.json) console.log(JSON.stringify({ summary: totals }));
+	else {
+		console.log(`\n${totalsLine(totals)}`);
+		if (values.junit) console.log(`  ${dim(`JUnit report: ${values.junit}`)}`);
+		console.log(`  ${cyan("➜")} bun run evalkit dev        ${dim("to inspect")}`);
+		if (project.config.share) console.log(`  ${cyan("➜")} bun run evalkit share      ${dim("to publish")}`);
+		console.log("");
+	}
+	if (controller.signal.aborted) process.exitCode = 130;
+	else if (summary.failed || broken.length) process.exitCode = 1;
 }
 //#endregion
 //#region ../cli/src/tidy.ts
@@ -9083,7 +9614,8 @@ function selectRuns(runs, head, options) {
 		otherCommits: 0,
 		dirty: 0,
 		archived: 0,
-		running: 0
+		running: 0,
+		cancelled: 0
 	};
 	for (const run of runs) {
 		const { source } = run;
@@ -9092,6 +9624,7 @@ function selectRuns(runs, head, options) {
 		else if (source.dirty && !options.allowDirty) result.dirty += 1;
 		else if (run.archived) result.archived += 1;
 		else if (run.status === "running") result.running += 1;
+		else if (run.status === "cancelled") result.cancelled += 1;
 		else result.selected.push({
 			...run,
 			source
@@ -9105,6 +9638,7 @@ function selectionSummary(selection, options) {
 		`${plural(selection.selected.length, "run")} to share`,
 		selection.otherCommits && !options.all ? `${selection.otherCommits} from other commits` : "",
 		selection.running ? `${selection.running} still running skipped` : "",
+		selection.cancelled ? `${selection.cancelled} cancelled skipped` : "",
 		selection.dirty ? `${selection.dirty} dirty skipped` : "",
 		selection.untracked ? `${selection.untracked} untracked skipped` : "",
 		selection.archived ? `${selection.archived} archived skipped` : ""
@@ -9152,29 +9686,61 @@ async function runBundle(options) {
 	}, { workspaceFiles: options.workspaceFiles });
 	return encodeBundle(files);
 }
-/** The server refused the bundle (4xx): retrying the same bundle cannot help. */
+/** The server refused this bundle (4xx): retrying it cannot help. */
 var UploadRejected = class extends Error {};
+/**
+* The URL does not answer as an EvalKit share server (a redirect, an HTML page,
+* an auth proxy). Every upload would fail the same way, so sharing stops.
+*/
+var ShareServerError = class extends Error {};
+/** `share.headers` from the config: called once, checked, never printed. */
+async function resolveShareHeaders(headers) {
+	let value;
+	try {
+		value = typeof headers === "function" ? await headers() : headers ?? {};
+	} catch (error) {
+		throw new Error(`share.headers failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("share.headers must be an object of header names to values");
+	const entries = Object.entries(value);
+	for (const [name, item] of entries) if (typeof item !== "string" || !item) throw new Error(`share.headers "${name}" must be a non-empty string`);
+	return Object.fromEntries(entries);
+}
+/** Check that the URL is an EvalKit share server and the headers get through. */
+async function checkShareServer(serverUrl, headers) {
+	const response = await fetch(new URL("/api/v1/repos", serverUrl), {
+		headers,
+		redirect: "manual"
+	});
+	await expectShareServer(response, serverUrl);
+	await response.body?.cancel();
+}
 /** POST one bundle, retrying network errors and 5xx responses. */
 async function uploadBundle(serverUrl, bundle, options = {}) {
 	const attempts = options.attempts ?? 3;
 	let lastError;
 	for (let attempt = 1; attempt <= attempts; attempt++) {
 		try {
-			return await postBundle(serverUrl, bundle);
+			return await postBundle(serverUrl, bundle, options.headers ?? {});
 		} catch (error) {
-			if (error instanceof UploadRejected) throw error;
+			if (error instanceof UploadRejected || error instanceof ShareServerError) throw error;
 			lastError = error;
 		}
-		if (attempt < attempts) await Bun.sleep((options.delayMs ?? 500) * 2 ** (attempt - 1));
+		if (attempt < attempts) await sleep((options.delayMs ?? 500) * 2 ** (attempt - 1));
 	}
 	throw lastError instanceof Error ? lastError : /* @__PURE__ */ new Error(`Upload failed: ${String(lastError)}`);
 }
-async function postBundle(serverUrl, bundle) {
+async function postBundle(serverUrl, bundle, headers) {
 	const response = await fetch(new URL("/api/v1/uploads", serverUrl), {
 		method: "POST",
-		headers: { "content-type": "application/octet-stream" },
-		body: bundle
+		headers: {
+			...headers,
+			"content-type": "application/octet-stream"
+		},
+		body: bundle,
+		redirect: "manual"
 	});
+	await expectShareServer(response, serverUrl);
 	if (response.ok) return parseUploadResult(await response.json());
 	const reason = await response.text();
 	if (response.status < 500) throw new UploadRejected(`Share server rejected the upload: ${reason}`);
@@ -9186,7 +9752,18 @@ function parseUploadResult(value) {
 		status: result.status,
 		url: result.url
 	};
-	throw new UploadRejected(`Unexpected share server response: ${JSON.stringify(value)}`);
+	throw new ShareServerError(`Unexpected share server response: ${JSON.stringify(value)}`);
+}
+/** Redirects and non-JSON answers come from something in front of the server. */
+async function expectShareServer(response, serverUrl) {
+	if (response.status >= 300 && response.status < 400) {
+		const location = response.headers.get("location");
+		const target = location ? new URL(location, serverUrl).host : "elsewhere";
+		throw new ShareServerError(`${new URL(serverUrl).host} redirected to ${target} instead of answering. If an auth proxy is in front of the share server, set share.headers in evalkit.config.ts.`);
+	}
+	const type = response.headers.get("content-type") ?? "";
+	const denied = response.status === 401 || response.status === 403;
+	if (denied || response.ok && !type.includes("json")) throw new ShareServerError(`${new URL(serverUrl).host} did not answer as an EvalKit share server (${response.status}${type ? `, ${type.split(";")[0]}` : ""}). ` + (denied ? "Check share.headers." : "Check share.url."));
 }
 async function shareRuns(options) {
 	const reports = createDashboardReportReader(options.reportRoot, options.registry);
@@ -9207,10 +9784,11 @@ async function shareRuns(options) {
 					run,
 					workspaceFiles: options.workspaceFiles
 				});
-				const uploaded = await uploadBundle(options.serverUrl, bundle);
+				const uploaded = await uploadBundle(options.serverUrl, bundle, { headers: options.headers });
 				result.bytes += bundle.byteLength;
 				result[uploaded.status] += 1;
 			} catch (error) {
+				if (error instanceof ShareServerError) throw error;
 				result.failed.push({
 					runId: run.id,
 					error: error instanceof Error ? error.message : String(error)
@@ -9227,7 +9805,7 @@ function formatBytes(size) {
 }
 //#endregion
 //#region ../cli/src/new-project-template.ts
-/** Provider-free example: one fixture, agent, scorer, and a two-cell matrix. */
+/** Provider-free example: one dataset, agent, scorer, and a two-cell matrix. */
 const starterFiles = {
 	"tsconfig.json": `${JSON.stringify({
 		compilerOptions: {
@@ -9240,15 +9818,17 @@ const starterFiles = {
 		},
 		include: [
 			"agents/**/*.ts",
-			"evals/**/*.ts",
+			"datasets/**/*.ts",
 			"judges/**/*.ts",
+			"scenarios/**/*.ts",
 			"evalkit.config.ts"
 		]
 	}, null, 2)}\n`,
 	"evalkit.config.ts": `import { defineConfig } from '@leostera/evalkit';
 
 export default defineConfig({
-  // Default-exported evals in evals/*.eval.ts are discovered automatically.
+  // Default-exported evals in scenarios/**/*.eval.ts are discovered automatically;
+  // each directory below scenarios/ is a suite.
   matrix: {
     id: 'styles',
     parameters: { style: ['plain', 'shout'] },
@@ -9256,7 +9836,16 @@ export default defineConfig({
   execution: { concurrency: 2, maxCells: 20 },
 });
 `,
-	"fixtures/greeting.txt": "Hello\n",
+	"datasets/greeting/greeting.txt": "Hello\n",
+	"datasets/greeting.dataset.ts": `import { defineDataset } from '@leostera/evalkit';
+
+// Files the agent can read: copied into each trial's workspace under greeting/.
+// Paths are relative to the project root.
+export const greeting = defineDataset({
+  id: 'greeting',
+  files: 'datasets/greeting',
+});
+`,
 	"agents/greeting-agent.ts": `import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { defineAgent } from '@leostera/evalkit';
@@ -9264,23 +9853,18 @@ import { defineAgent } from '@leostera/evalkit';
 // Replace this in-process example with an adapter for your own agent.
 export const greetingAgent = defineAgent({
   identity: { id: 'greeting-agent', kind: 'in-process' },
-  runtimes: { local: { kind: 'in-process' } },
-  async start({ context, onEvent }) {
+  async start({ context }) {
     return {
+      // Return the reply; EvalKit records it. Call onEvent instead to stream events.
       async send(message: string) {
-        // Fixtures copied to the candidate workspace are visible to the AUT.
-        const prefix = (await readFile(join(context.workspace.root, 'greeting.txt'), 'utf8')).trim();
+        const file = join(context.workspace.root, 'greeting', 'greeting.txt');
+        const prefix = (await readFile(file, 'utf8')).trim();
         const style = context.parameters?.style;
         if (style !== 'plain' && style !== 'shout')
           throw new Error(\`Unknown style: \${String(style)}\`);
         const greeting = \`\${prefix}, \${message}!\`;
-        await onEvent({
-          kind: 'message', role: 'assistant',
-          content: style === 'shout' ? greeting.toUpperCase() : greeting,
-          timestamp: new Date().toISOString(),
-        });
+        return style === 'shout' ? greeting.toUpperCase() : greeting;
       },
-      async close() {},
     };
   },
 });
@@ -9288,25 +9872,20 @@ export const greetingAgent = defineAgent({
 	"judges/matches-greeting.ts": `import { predicate } from '@leostera/evalkit';
 
 // Deterministic scorers need no judge model; add a separate judge agent for judge(...) rules.
-export const matchesGreeting = predicate('matches greeting and style', ({ context, trajectory }) => {
-  const reply = trajectory.events.filter(
-    (event) => event.source === 'aut' && event.kind === 'message' && event.role === 'assistant',
-  ).at(-1);
+export const matchesGreeting = predicate('matches greeting and style', ({ context, output }) => {
   const expected = context.parameters?.style === 'shout' ? 'HELLO, ADA!' : 'Hello, Ada!';
-  return {
-    value: reply?.kind === 'message' && reply.content === expected ? 1 : 0,
-    explanation: \`Expected \${expected}\`,
-  };
+  return { value: output === expected ? 1 : 0, explanation: \`Expected \${expected}\` };
 });
 `,
-	"evals/greeting.eval.ts": `import { defineEval, file, user } from '@leostera/evalkit';
+	"scenarios/greeting.eval.ts": `import { defineEval, user } from '@leostera/evalkit';
 import { greetingAgent } from '../agents/greeting-agent.js';
+import { greeting } from '../datasets/greeting.dataset.js';
 import { matchesGreeting } from '../judges/matches-greeting.js';
 
 export default defineEval({
   id: 'greeting',
   agent: greetingAgent,
-  fixtures: [file('fixtures/greeting.txt', { dst: 'greeting.txt', visibility: 'candidate' })],
+  datasets: [greeting],
   transcript: [user('Ada')],
   scoring: [matchesGreeting],
 });
@@ -9315,10 +9894,11 @@ export default defineEval({
 function starterReadme(name) {
 	return `# ${name}
 
-A local EvalKit project. Each default-exported eval in \`evals/\` is discovered automatically.
-Start with \`evals/greeting.eval.ts\`: it imports an in-process agent from \`agents/\`,
-a deterministic scorer from \`judges/\`, and a candidate-visible file from \`fixtures/\`.
-Replace these examples with your own agent, tasks, and checks.
+A local EvalKit project. Each default-exported eval in \`scenarios/\` is discovered
+automatically, and each directory below \`scenarios/\` is a suite. Start with
+\`scenarios/greeting.eval.ts\`: it imports an in-process agent from \`agents/\`, a
+dataset of files from \`datasets/\`, and a deterministic scorer from \`judges/\`.
+Keep each scenario small and concrete; share agents, datasets and judges between them.
 
 \`evalkit.config.ts\` defines a provider-free \`styles\` matrix; the example agent reads
 \`context.parameters.style\`. Run:
@@ -9332,6 +9912,9 @@ bun run evalkit dev                             # inspect local reports and laun
 bun run evalkit build                           # write a read-only, shareable snapshot
 \`\`\`
 
+Coding agents can use the EvalKit skills that ship with the package, in
+\`node_modules/@leostera/evalkit/dist/skills/\` (start with \`evalkit/SKILL.md\`).
+
 EvalKit installs from npm. Reports, trial workspaces, and dashboard snapshots stay local
 and are ignored by Git. A snapshot in \`_evalkit-dashboard/\` contains report messages,
 scores, evidence, and candidate files (never evaluator workspaces): review it before sharing.
@@ -9339,7 +9922,7 @@ scores, evidence, and candidate files (never evaluator workspaces): review it be
 }
 //#endregion
 //#region ../cli/src/new-project.ts
-const npmDependency = "^0.4.0";
+const npmDependency = "^0.6.0";
 const ignored = [
 	"node_modules/",
 	"_evalkit-results/",
@@ -9403,9 +9986,9 @@ async function checkExistingScaffold(root) {
 	for (const extension of ["js", "mjs"]) if (await exists(resolve(root, `evalkit.config.${extension}`))) throw new Error(`EvalKit config already exists: evalkit.config.${extension}`);
 	for (const directory of [
 		"agents",
-		"fixtures",
-		"evals",
-		"judges"
+		"datasets",
+		"judges",
+		"scenarios"
 	]) {
 		const path = resolve(root, directory);
 		if (await exists(path) && !(await stat(path)).isDirectory()) throw new Error(`Scaffold directory is not a directory: ${path}`);
@@ -9451,8 +10034,300 @@ async function writeStarterReadme(root, name, inPlace) {
 	if (!inPlace && !await exists(readmePath)) await writeFile(readmePath, starterReadme(name));
 }
 //#endregion
+//#region ../cli/src/compare.ts
+const FINISHED = /* @__PURE__ */ new Set([
+	"passed",
+	"failed",
+	"errored"
+]);
+function unitKey(run) {
+	return `${run.evalId}\0${canonicalParameters(run.parameters ?? {})}`;
+}
+/** The run that represents each unit: the latest finished one, else the latest at all. */
+function latestByUnit(runs) {
+	const byUnit = /* @__PURE__ */ new Map();
+	let ignored = 0;
+	const ordered = [...runs].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+	for (const run of ordered) {
+		if (run.status === "running") continue;
+		const key = unitKey(run);
+		const current = byUnit.get(key);
+		if (!current) byUnit.set(key, run);
+		else {
+			ignored += 1;
+			if (!FINISHED.has(current.status) && FINISHED.has(run.status)) byUnit.set(key, run);
+		}
+	}
+	return {
+		byUnit,
+		ignored
+	};
+}
+function scorerSignature(run) {
+	return (run.scorers ?? []).map((scorer) => `${scorer.kind}:${scorer.name}`).sort().join("\n");
+}
+function classify(base, head) {
+	if (!base) return { change: "new" };
+	if (!head) return { change: "removed" };
+	for (const [side, run] of [["base", base], ["head", head]]) if (!FINISHED.has(run.status) || run.status === "errored") return {
+		change: "not-comparable",
+		reason: `${run.status} on ${side}`
+	};
+	if (scorerSignature(base) !== scorerSignature(head)) return {
+		change: "not-comparable",
+		reason: "different scorers"
+	};
+	if (base.status === "passed" && head.status === "failed") return { change: "regressed" };
+	if (base.status === "failed" && head.status === "passed") return { change: "fixed" };
+	return { change: "unchanged" };
+}
+function label(run, suite) {
+	const parameters = Object.entries(run.parameters ?? {}).map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`).join(" ");
+	const name = suite === "default" ? run.evalId : `${suite}/${run.evalId}`;
+	return parameters ? `${name} [${parameters}]` : name;
+}
+function mean(values) {
+	return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : void 0;
+}
+function suiteScorers(units) {
+	const sides = {
+		base: /* @__PURE__ */ new Map(),
+		head: /* @__PURE__ */ new Map()
+	};
+	const kinds = /* @__PURE__ */ new Map();
+	for (const unit of units) {
+		if (unit.change === "not-comparable" || !unit.base || !unit.head) continue;
+		for (const side of ["base", "head"]) for (const scorer of unit[side].scorers ?? []) {
+			const key = `${scorer.kind}\0${scorer.name}`;
+			kinds.set(key, {
+				name: scorer.name,
+				kind: scorer.kind
+			});
+			if (scorer.mean === void 0) continue;
+			sides[side].set(key, [...sides[side].get(key) ?? [], scorer.mean]);
+		}
+	}
+	for (const unit of units) {
+		if (!unit.base || !unit.head) continue;
+		const names = (run) => new Set((run.scorers ?? []).map((s) => `${s.kind}\0${s.name}`));
+		const baseNames = names(unit.base);
+		const headNames = names(unit.head);
+		for (const scorer of unit.base.scorers ?? []) if (!headNames.has(`${scorer.kind}\0${scorer.name}`)) kinds.set(`${scorer.kind}\0${scorer.name}`, {
+			name: scorer.name,
+			kind: scorer.kind,
+			onlyOn: "base"
+		});
+		for (const scorer of unit.head.scorers ?? []) if (!baseNames.has(`${scorer.kind}\0${scorer.name}`)) kinds.set(`${scorer.kind}\0${scorer.name}`, {
+			name: scorer.name,
+			kind: scorer.kind,
+			onlyOn: "head"
+		});
+	}
+	return [...kinds].map(([key, scorer]) => {
+		const base = mean(sides.base.get(key) ?? []);
+		const head = mean(sides.head.get(key) ?? []);
+		return {
+			...scorer,
+			...base === void 0 || scorer.onlyOn ? {} : { base },
+			...head === void 0 || scorer.onlyOn ? {} : { head }
+		};
+	}).sort((a, b) => a.name.localeCompare(b.name));
+}
+/**
+* Matches base and head runs by eval ID and parameters and classifies each
+* unit. It reads recorded verdicts and means; it never re-scores.
+*/
+function compareRuns(baseRuns, headRuns) {
+	const base = latestByUnit(baseRuns);
+	const head = latestByUnit(headRuns);
+	const units = [.../* @__PURE__ */ new Set([...base.byUnit.keys(), ...head.byUnit.keys()])].map((key) => {
+		const baseRun = base.byUnit.get(key);
+		const headRun = head.byUnit.get(key);
+		const run = headRun ?? baseRun;
+		const suite = headRun?.suiteId ?? baseRun?.suiteId ?? "default";
+		return {
+			label: label(run, suite),
+			evalId: run.evalId,
+			suite,
+			parameters: run.parameters ?? {},
+			...classify(baseRun, headRun),
+			...baseRun ? { base: baseRun } : {},
+			...headRun ? { head: headRun } : {}
+		};
+	}).sort((a, b) => a.label.localeCompare(b.label));
+	const changedDatasets = /* @__PURE__ */ new Set();
+	for (const unit of units) {
+		if (!unit.base || !unit.head) continue;
+		const digests = (run) => new Map((run.datasets ?? []).map((d) => [d.id, d.digests.join(",")]));
+		const before = digests(unit.base);
+		const after = digests(unit.head);
+		for (const id of /* @__PURE__ */ new Set([...before.keys(), ...after.keys()])) if (before.get(id) !== after.get(id)) changedDatasets.add(id);
+	}
+	const bySuite = /* @__PURE__ */ new Map();
+	for (const unit of units) bySuite.set(unit.suite, [...bySuite.get(unit.suite) ?? [], unit]);
+	const suites = [...bySuite].map(([suite, members]) => {
+		const side = (pick) => {
+			const runs = members.flatMap((unit) => unit[pick] ? [unit[pick]] : []);
+			return {
+				passed: runs.filter((run) => run.status === "passed").length,
+				total: runs.length
+			};
+		};
+		return {
+			suite,
+			base: side("base"),
+			head: side("head"),
+			scorers: suiteScorers(members)
+		};
+	}).sort((a, b) => a.suite.localeCompare(b.suite));
+	const counts = {
+		regressed: 0,
+		fixed: 0,
+		new: 0,
+		removed: 0,
+		unchanged: 0,
+		"not-comparable": 0
+	};
+	for (const unit of units) counts[unit.change] += 1;
+	return {
+		units,
+		suites,
+		changedDatasets: [...changedDatasets].sort(),
+		ignoredRuns: {
+			base: base.ignored,
+			head: head.ignored
+		},
+		counts
+	};
+}
+function parseFailOn(values) {
+	return values.flatMap((value) => value.split(",")).map((value) => {
+		if (value === "regression") return { kind: "regression" };
+		const drop = /^score-drop=(\d*\.?\d+)$/.exec(value);
+		if (drop) {
+			const threshold = Number(drop[1]);
+			if (threshold >= 0 && threshold <= 1) return {
+				kind: "score-drop",
+				threshold
+			};
+		}
+		throw new Error(`Unknown --fail-on ${JSON.stringify(value)}; use regression or score-drop=0.05`);
+	});
+}
+/** Human-readable reasons the comparison fails the given conditions; empty when it passes. */
+function gateFailures(comparison, conditions) {
+	const failures = [];
+	for (const condition of conditions) {
+		if (condition.kind === "regression" && comparison.counts.regressed) failures.push(`${comparison.counts.regressed} regressed: ${comparison.units.filter((unit) => unit.change === "regressed").map((unit) => unit.label).join(", ")}`);
+		if (condition.kind === "score-drop") {
+			for (const suite of comparison.suites) for (const scorer of suite.scorers) if (scorer.base !== void 0 && scorer.head !== void 0 && scorer.base - scorer.head > condition.threshold) failures.push(`${suite.suite} ${scorer.name} dropped ${scorer.base.toFixed(2)} → ${scorer.head.toFixed(2)} (more than ${condition.threshold})`);
+		}
+	}
+	return failures;
+}
+const fixed2 = (value) => value === void 0 ? "–" : value.toFixed(2);
+function scorerText(scorer) {
+	if (scorer.onlyOn) return `${scorer.name} only on ${scorer.onlyOn}`;
+	return `${scorer.name} ${fixed2(scorer.base)} → ${fixed2(scorer.head)}`;
+}
+/** A merge request comment. */
+function comparisonMarkdown(comparison, heading) {
+	const { counts } = comparison;
+	const list = (change) => comparison.units.filter((unit) => unit.change === change).map((unit) => `- \`${unit.label}\`${unit.reason ? ` (${unit.reason})` : ""}`);
+	const lines = [
+		`### EvalKit: ${heading.base} → ${heading.head}`,
+		"",
+		`**${counts.regressed} regressed** · ${counts.fixed} fixed · ${counts.new} new · ${counts.removed} removed · ${counts.unchanged} unchanged${counts["not-comparable"] ? ` · ${counts["not-comparable"]} not comparable` : ""}`,
+		""
+	];
+	for (const [title, change] of [
+		["Regressed", "regressed"],
+		["Fixed", "fixed"],
+		["New", "new"],
+		["Removed", "removed"],
+		["Not comparable", "not-comparable"]
+	]) {
+		const items = list(change);
+		if (items.length) lines.push(`**${title}**`, "", ...items, "");
+	}
+	lines.push("| Suite | Passed | Scores |", "| --- | --- | --- |");
+	for (const suite of comparison.suites) lines.push(`| ${suite.suite} | ${suite.base.passed}/${suite.base.total} → ${suite.head.passed}/${suite.head.total} | ${suite.scorers.map(scorerText).join("<br>") || "–"} |`);
+	if (comparison.changedDatasets.length) lines.push("", `Datasets changed: ${comparison.changedDatasets.map((id) => `\`${id}\``).join(", ")}`);
+	return `${lines.join("\n")}\n`;
+}
+function comparisonLines(comparison) {
+	const { counts } = comparison;
+	const labels = (change) => comparison.units.filter((unit) => unit.change === change).map((unit) => unit.reason ? `${unit.label} (${unit.reason})` : unit.label);
+	return {
+		units: [
+			"regressed",
+			"fixed",
+			"not-comparable"
+		].map((change) => ({
+			change,
+			labels: labels(change)
+		})).filter((entry) => entry.labels.length),
+		counts: `+ ${counts.new} new   − ${counts.removed} removed   ${counts.unchanged} unchanged`,
+		suites: comparison.suites.map((suite) => `${suite.suite.padEnd(12)} ${suite.base.passed}/${suite.base.total} → ${suite.head.passed}/${suite.head.total}   ${suite.scorers.map(scorerText).join(" · ")}`),
+		...comparison.changedDatasets.length ? { datasets: `datasets    ${comparison.changedDatasets.join(", ")} changed` } : {}
+	};
+}
+//#endregion
+//#region ../cli/src/compare-command.ts
+/** The runs at HEAD that `compare` uses: not running, not archived, clean unless allowed. */
+function headRuns(runs, head, options) {
+	return runs.filter((run) => run.source !== void 0 && run.source.commit === head.commit && repoKey(run.source) === repoKey(head) && (!run.source.dirty || options.allowDirty) && !run.archived && run.status !== "running");
+}
+/** Base runs from a results directory, e.g. an artifact from the base pipeline. Dirty runs are never a base. */
+async function baseRunsFromDirectory(directory, registry) {
+	return (await createDashboardReportReader(directory, registry).listRuns()).filter((run) => !run.source?.dirty && !run.archived);
+}
+/** A full SHA, a unique prefix of at least 7 characters, or a ref's latest shared commit. */
+function resolveSharedRef(commits, ref) {
+	if (/^[0-9a-f]{40}$/.test(ref)) return commits.some((row) => row.commit === ref) ? ref : void 0;
+	if (/^[0-9a-f]{7,39}$/.test(ref)) {
+		const matches = commits.filter((row) => row.commit.startsWith(ref));
+		if (matches.length === 1) return matches[0].commit;
+	}
+	return commits.find((row) => row.ref === ref)?.commit;
+}
+async function getJson(url, serverUrl, headers) {
+	const response = await fetch(url, {
+		headers,
+		redirect: "manual"
+	});
+	await expectShareServer(response, serverUrl);
+	if (response.status === 404) {
+		await response.body?.cancel();
+		return;
+	}
+	if (!response.ok) throw new Error(`Share server error ${response.status}: ${await response.text()}`);
+	return await response.json();
+}
+/** Base runs shared for `ref` of this repository; `undefined` when the base was never shared. */
+async function baseRunsFromServer(options) {
+	const repo = repoKey(options.head);
+	const api = (path) => new URL(`/api/v1/repos/${repo}/-/${path}`, options.serverUrl);
+	const listed = await getJson(api("commits"), options.serverUrl, options.headers);
+	const commits = Array.isArray(listed?.commits) ? listed.commits : [];
+	const commit = resolveSharedRef(commits, options.ref);
+	if (!commit) return void 0;
+	const data = await getJson(api(`${commit}/data/runs.json`), options.serverUrl, options.headers);
+	const runs = (Array.isArray(data?.runs) ? data.runs : []).filter(isComparedRun);
+	const ref = commits.find((row) => row.commit === commit)?.ref;
+	return {
+		commit,
+		...ref ? { ref } : {},
+		runs: runs.filter((run) => !run.source?.dirty)
+	};
+}
+/** Shared runs are uploaded by other EvalKit versions; keep only what compare can read. */
+function isComparedRun(value) {
+	const run = value;
+	return typeof run === "object" && run !== null && typeof run.id === "string" && typeof run.evalId === "string" && typeof run.status === "string" && typeof run.startedAt === "string";
+}
+//#endregion
 //#region ../cli/src/index.ts
-const VERSION = "0.4.0";
 let projectRoot = process.cwd();
 let reportRoot = resolve(projectRoot, "_evalkit-results");
 const sandboxRoot = resolve(projectRoot, "_evalkit-sandbox");
@@ -9466,7 +10341,7 @@ function runtimeFor(evaluation) {
 		"local",
 		"remote",
 		"sandbox"
-	]) if (evaluation.agent.runtimes?.[runtime]) return runtime;
+	]) if (agentRuntimes(evaluation.agent)[runtime]) return runtime;
 }
 async function runEvals(options) {
 	const registry = await loadRegistry();
@@ -9495,7 +10370,7 @@ async function runProvenance(registry) {
 	] });
 	return {
 		...source ? { source } : {},
-		catalog: runCatalog(registry)
+		catalog: runCatalog(registry, projectRoot)
 	};
 }
 function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
@@ -9577,7 +10452,37 @@ function registerMatrixRoutes(app, registry, dashboardMatrix, trials) {
 	});
 }
 function registerReportRoutes(app, registry, reports) {
-	app.get("/v1/catalog", (context) => context.json({ evals: registry.catalog() }));
+	app.get("/v1/project", async (context) => {
+		const source = await detectSource(projectRoot, { outputDirs: [
+			reportRoot,
+			sandboxRoot,
+			resolve(projectRoot, "_evalkit-dashboard")
+		] });
+		return context.json({ project: source ? projectInfo(source) : null });
+	});
+	app.get("/v1/datasets/:id/files", async (context) => {
+		const dataset = projectDatasets(registry).get(context.req.param("id"));
+		if (!dataset) return context.text("Unknown dataset", 404);
+		try {
+			return context.json(await datasetListing(dataset, projectRoot));
+		} catch (error) {
+			return context.text(error instanceof Error ? error.message : "Unable to list dataset", 400);
+		}
+	});
+	app.get("/v1/datasets/:id/files/:path{.+}", async (context) => {
+		const dataset = projectDatasets(registry).get(context.req.param("id"));
+		if (!dataset) return context.text("Unknown dataset", 404);
+		const file = context.req.param("path");
+		try {
+			return new Response(await staticDatasetFile(dataset, projectRoot, file), { headers: { "content-type": fileContentType(file) } });
+		} catch (error) {
+			return context.text(error instanceof Error ? error.message : "Unable to read dataset file", 404);
+		}
+	});
+	app.get("/v1/catalog", (context) => context.json({
+		evals: registry.catalog({ root: projectRoot }),
+		loadErrors: project?.loadErrors ?? []
+	}));
 	app.get("/v1/evals", (context) => context.json({ evals: registry.metadata() }));
 	app.get("/v1/suites", (context) => context.json({ suites: registry.suiteMetadata() }));
 	app.get("/v1/runs", async (context) => context.json({ runs: await reports.listRuns() }));
@@ -9602,9 +10507,9 @@ function registerReportRoutes(app, registry, reports) {
 			return context.text(error instanceof Error ? error.message : "Unable to list workspace", 400);
 		}
 	});
-	app.get("/v1/runs/:runId/trials/:trialId/workspace/*", async (context) => {
+	app.get("/v1/runs/:runId/trials/:trialId/workspace/:path{.+}", async (context) => {
 		try {
-			return await reports.candidateFile(context.req.param("runId"), context.req.param("trialId"), context.req.param("*") ?? "");
+			return await reports.candidateFile(context.req.param("runId"), context.req.param("trialId"), context.req.param("path"));
 		} catch (error) {
 			return context.text(error instanceof Error ? error.message : "Unable to read workspace file", 400);
 		}
@@ -9630,12 +10535,12 @@ function registerDashboardAssets(app, dashboardRoot) {
 		const pathname = url.pathname === "/" ? "/index.html" : url.pathname;
 		const file = resolve(dashboardRoot, `.${pathname}`);
 		if (!file.startsWith(dashboardRoot)) return context.text("Not found", 404);
-		let content = Bun.file(file);
 		let servedFile = file;
-		if (!await content.exists()) {
+		let content = await readFile(file).catch(() => void 0);
+		if (!content) {
 			if (extname(pathname)) return context.text("Not found", 404);
 			servedFile = resolve(dashboardRoot, "index.html");
-			content = Bun.file(servedFile);
+			content = await readFile(servedFile);
 		}
 		return new Response(content, { headers: { "content-type": fileContentType(servedFile) } });
 	});
@@ -9663,10 +10568,7 @@ async function serveDashboard() {
 	const port = Number(process.env.PORT ?? 4317);
 	let server;
 	try {
-		server = Bun.serve({
-			port,
-			fetch: app.fetch
-		});
+		server = await serve(app.fetch, port);
 	} catch (error) {
 		if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Stop the other server or choose another port: PORT=4318 evalkit dev`);
 		throw error;
@@ -9678,19 +10580,12 @@ async function serveDashboard() {
 	const matrixCells = dashboardMatrix?.count({ parameters: {} });
 	const row = (label, value) => console.log(`  ${cyan("➜")}  ${bold(label.padEnd(9))}${value}`);
 	banner("dev", VERSION);
+	printLoadErrors(project?.loadErrors ?? []);
 	row("Local:", cyan(url));
 	row("Project:", `${plural(registry.evals.length, "eval")} · ${plural(registry.suites.length, "suite")}` + (dashboardMatrix ? ` · matrix ${authoringId(dashboardMatrix)} (${plural(matrixCells, "cell")})` : ""));
 	row("Reports:", `${relative(process.cwd(), reportRoot) || "."} ${dim(`(${plural(runs, "run")}${legacy ? `, ${legacy} from an older EvalKit hidden` : ""})`)}`);
 	console.log(`\n  ${dim("Runs started from the dashboard are logged below. Press Ctrl+C to stop.")}\n`);
-	if (process.env.EVALKIT_NO_OPEN !== "1") {
-		const opener = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
-		try {
-			Bun.spawn([opener, url], {
-				stdout: "ignore",
-				stderr: "ignore"
-			});
-		} catch {}
-	}
+	if (process.env.EVALKIT_NO_OPEN !== "1") openInBrowser(url);
 }
 const config = text("config").pipe(optional);
 const runOptions = {
@@ -9710,7 +10605,10 @@ const runOptions = {
 	json: boolean("json"),
 	dryRun: boolean("dry-run"),
 	all: boolean("all"),
-	local: boolean("local")
+	local: boolean("local"),
+	junit: text("junit").pipe(optional),
+	verbose: boolean("verbose"),
+	skipBroken: boolean("skip-broken")
 };
 function runInput(options, positionals) {
 	const unexpected = positionals.find((value) => value.startsWith("-"));
@@ -9732,7 +10630,10 @@ function runInput(options, positionals) {
 		json: options.json,
 		"dry-run": options.dryRun,
 		all: options.all,
-		local: options.local
+		local: options.local,
+		junit: Option.getOrUndefined(options.junit),
+		verbose: options.verbose,
+		"skip-broken": options.skipBroken
 	}, positionals);
 }
 const execute = (input) => Effect.tryPromise({
@@ -9775,6 +10676,12 @@ const buildCommand = make$1("build", {
 		})) await runAllEvals(configPath);
 		const result = await buildDashboard({
 			registry: project.registry,
+			root: projectRoot,
+			source: await detectSource(projectRoot, { outputDirs: [
+				reportRoot,
+				sandboxRoot,
+				resolve(projectRoot, "_evalkit-dashboard")
+			] }),
 			reportRoot,
 			dashboardRoot: dashboardAssets(),
 			outDir: resolve(projectRoot, Option.getOrElse(out, () => "_evalkit-dashboard"))
@@ -9791,8 +10698,13 @@ const buildCommand = make$1("build", {
 async function loadProjectStep(configPath) {
 	await step("Eval project", () => useProject(configPath), () => {
 		const where = relative(process.cwd(), projectRoot);
-		return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}`;
+		const broken = project.loadErrors.length;
+		return `${plural(project.registry.evals.length, "eval")} loaded${where ? ` from ${where}` : ""}${broken ? yellow(` · ${plural(broken, "file")} failed to load`) : ""}`;
 	});
+	if (project.loadErrors.length) {
+		console.log("");
+		printLoadErrors(project.loadErrors);
+	}
 }
 /**
 * A snapshot needs data: with no current results, run the evals. Otherwise ask,
@@ -9850,10 +10762,7 @@ const shareServeCommand = make$1("serve", {
 		});
 		let server;
 		try {
-			server = Bun.serve({
-				port,
-				fetch: app.fetch
-			});
+			server = await serve(app.fetch, port);
 		} catch (error) {
 			if (error.code === "EADDRINUSE") throw new Error(`Port ${port} is already in use. Choose another: evalkit share serve --port 4319`);
 			throw error;
@@ -9891,6 +10800,14 @@ const shareCommand = make$1("share", {
 		if (!selection.selected.length) throw new Error(noRunsMessage(selection, head, { all }));
 		const result = await shareRuns({
 			serverUrl,
+			headers: await step("Server", async () => {
+				const headers = await resolveShareHeaders(project.config.share?.headers);
+				await checkShareServer(serverUrl, headers);
+				return headers;
+			}, (headers) => {
+				const names = Object.keys(headers);
+				return `${new URL(serverUrl).host}${names.length ? dim(` with ${names.join(", ")}`) : ""}`;
+			}),
 			reportRoot,
 			registry,
 			runs: selection.selected,
@@ -9905,13 +10822,118 @@ const shareCommand = make$1("share", {
 	},
 	catch: (error) => error
 })).pipe(withSubcommands([shareServeCommand]));
+const compareCommand = make$1("compare", {
+	config,
+	base: text("base").pipe(optional),
+	baseDir: text("base-dir").pipe(optional),
+	markdown: text("markdown").pipe(optional),
+	json: boolean("json"),
+	failOn: text("fail-on").pipe(repeated),
+	requireBase: boolean("require-base"),
+	allowDirty: boolean("allow-dirty")
+}, (options) => Effect.tryPromise({
+	try: () => compareCommandAction({
+		config: Option.getOrUndefined(options.config),
+		base: Option.getOrUndefined(options.base),
+		baseDir: Option.getOrUndefined(options.baseDir),
+		markdown: Option.getOrUndefined(options.markdown),
+		json: options.json,
+		failOn: options.failOn,
+		requireBase: options.requireBase,
+		allowDirty: options.allowDirty
+	}),
+	catch: (error) => error
+}));
+async function compareCommandAction(options) {
+	if (Boolean(options.base) === Boolean(options.baseDir)) throw new Error("Give exactly one of --base <ref> or --base-dir <dir>");
+	const conditions = parseFailOn(options.failOn);
+	const say = options.json ? () => {} : (line = "") => console.log(line);
+	if (!options.json) banner("compare", VERSION);
+	await useProject(options.config);
+	const registry = project.registry;
+	const head = await detectSource(projectRoot, { outputDirs: [
+		reportRoot,
+		sandboxRoot,
+		resolve(projectRoot, "_evalkit-dashboard")
+	] });
+	if (!head) throw new Error("evalkit compare needs to know the commit: run it inside a git repository or set EVALKIT_SOURCE_REPO and EVALKIT_SOURCE_COMMIT.");
+	const headSide = headRuns(await createDashboardReportReader(reportRoot, registry).listRuns(), head, { allowDirty: options.allowDirty });
+	let base;
+	if (options.baseDir) {
+		const runs = await baseRunsFromDirectory(resolve(process.cwd(), options.baseDir), registry);
+		base = runs.length ? {
+			label: options.baseDir,
+			runs
+		} : void 0;
+	} else {
+		const serverUrl = process.env.EVALKIT_SHARE_URL ?? project.config.share?.url;
+		if (!serverUrl) throw new Error("No share server configured for --base. Set share: { url } in evalkit.config.ts or EVALKIT_SHARE_URL, or use --base-dir.");
+		const found = await baseRunsFromServer({
+			serverUrl,
+			headers: await resolveShareHeaders(project.config.share?.headers),
+			head,
+			ref: options.base
+		});
+		base = found?.runs.length ? {
+			label: `${found.commit.slice(0, 7)}${found.ref ? ` (${found.ref})` : ""}`,
+			runs: found.runs
+		} : void 0;
+	}
+	const headLabel = `${shortSource(head)}${head.ref ? ` (${head.ref})` : ""}`;
+	if (!base) {
+		const message = `No shared results for base ${options.base ?? options.baseDir}; nothing to compare.`;
+		if (options.json) console.log(JSON.stringify({
+			base: null,
+			head: headLabel,
+			message
+		}));
+		else say(`  ${yellow("◇")} ${message}\n`);
+		if (options.requireBase) process.exitCode = 1;
+		return;
+	}
+	if (!headSide.length) throw new Error(`No results at ${headLabel} to compare. Run \`evalkit run\` first${head.dirty ? ", or pass --allow-dirty" : ""}.`);
+	const comparison = compareRuns(base.runs, headSide);
+	const failures = gateFailures(comparison, conditions);
+	if (options.markdown) await writeFile(resolve(process.cwd(), options.markdown), comparisonMarkdown(comparison, {
+		base: base.label,
+		head: headLabel
+	}));
+	if (options.json) console.log(JSON.stringify({
+		base: base.label,
+		head: headLabel,
+		...comparison,
+		failures
+	}));
+	else {
+		const lines = comparisonLines(comparison);
+		say(`  base ${bold(base.label)} → head ${bold(headLabel)}\n`);
+		const marks = {
+			regressed: red("✗"),
+			fixed: green("✓"),
+			"not-comparable": yellow("◌")
+		};
+		for (const { change, labels } of lines.units) say(`  ${marks[change]} ${`${labels.length} ${change.replace("-", " ")}`.padEnd(18)} ${labels.join(", ")}`);
+		say(`  ${dim(lines.counts)}`);
+		const ignored = comparison.ignoredRuns.base + comparison.ignoredRuns.head;
+		if (ignored) say(`  ${dim(`${plural(ignored, "older run")} of the same scenarios ignored`)}`);
+		say("");
+		for (const line of lines.suites) say(`  ${line}`);
+		if (lines.datasets) say(`  ${lines.datasets}`);
+		say("");
+		for (const failure of failures) say(`  ${red("✗")} ${failure}`);
+		if (options.markdown) say(`  ${dim(`Markdown: ${options.markdown}`)}`);
+		if (failures.length || options.markdown) say("");
+	}
+	if (failures.length) process.exitCode = 1;
+}
 const cli = run(make$1("evalkit").pipe(withSubcommands([
 	newCommand,
 	runCommand,
 	devCommand,
 	buildCommand,
 	tidyCommand,
-	shareCommand
+	shareCommand,
+	compareCommand
 ])), {
 	name: "EvalKit",
 	version: VERSION
@@ -9920,6 +10942,7 @@ const argv = process.argv.slice();
 if (argv[2] === "help") argv.splice(2, 1, "--help");
 if (argv.at(-2) === "--" && argv.at(-1) === "--help") argv.splice(-2, 1);
 try {
+	assertSupportedRuntime();
 	await Effect.runPromise(cli(argv).pipe(Effect.provide(layer)));
 } catch (error) {
 	console.error(error instanceof Error ? error.message : String(error));
@@ -9927,5 +10950,3 @@ try {
 }
 //#endregion
 export {};
-
-//# sourceMappingURL=cli.mjs.map