@ls-stack/agent-eval 0.6.0 → 0.7.0

package/dist/{cli-C-n-Fd4o.mjs → runOrchestration-HaMahl6b.mjs}

@@ -1,16 +1,14 @@
+import { createRequire, registerHooks } from "node:module";
 import { createHash } from "node:crypto";
 import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
-import { dirname, extname, join, relative, resolve } from "node:path";
+import { extname, isAbsolute, join, relative, resolve } from "node:path";
 import { z, z as z$1 } from "zod/v4";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { Buffer as Buffer$1 } from "node:buffer";
 import { getCompositeKey } from "@ls-stack/utils/getCompositeKey";
-import { watch } from "chokidar";
-import { glob } from "glob";
 import { existsSync } from "node:fs";
 import { resultify } from "t-result";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { spawn, spawnSync } from "node:child_process";
 //#region ../sdk/src/defineEval.ts
 const evalRegistry = /* @__PURE__ */ new Map();
 /** Return the in-memory registry of evals defined in the current process. */
@@ -2230,80 +2228,6 @@ function safeJsonParse(text) {
 	return parsed.value;
 }
 //#endregion
-//#region ../runner/src/chartValidation.ts
-function isValidColumnMetric(metric, columnsByKey, evalId, warnings) {
-	const columnDef = columnsByKey.get(metric.key);
-	if (!columnDef) {
-		warnings.push(`[${evalId}] chart metric references unknown column "${metric.key}" — dropped`);
-		return false;
-	}
-	if (metric.aggregate === "passThresholdRate") {
-		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
-			warnings.push(`[${evalId}] chart metric "${metric.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
-			return false;
-		}
-	}
-	return true;
-}
-function isValidTooltipExtra(extra, columnsByKey, evalId, warnings) {
-	const columnDef = columnsByKey.get(extra.key);
-	if (!columnDef) {
-		warnings.push(`[${evalId}] chart tooltip extra references unknown column "${extra.key}" — dropped`);
-		return false;
-	}
-	if (extra.aggregate === "passThresholdRate") {
-		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
-			warnings.push(`[${evalId}] chart tooltip extra "${extra.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
-			return false;
-		}
-	}
-	return true;
-}
-function sanitizeChart(chart, columnsByKey, evalId, warnings) {
-	const metrics = chart.metrics.filter((metric) => {
-		if (metric.source === "builtin") return true;
-		return isValidColumnMetric(metric, columnsByKey, evalId, warnings);
-	});
-	if (metrics.length === 0) {
-		warnings.push(`[${evalId}] chart had no valid metrics after validation — chart dropped`);
-		return null;
-	}
-	const tooltipExtras = chart.tooltipExtras?.filter((extra) => {
-		if (extra.source === "builtin") return true;
-		return isValidTooltipExtra(extra, columnsByKey, evalId, warnings);
-	});
-	return {
-		...chart,
-		metrics,
-		tooltipExtras: tooltipExtras?.length ? tooltipExtras : void 0
-	};
-}
-/**
-* Validate and sanitize an authored `charts` config against the eval's
-* declared columns. Drops metrics/extras that reference unknown columns or
-* misuse `passThresholdRate`, and drops entire charts whose metrics are all
-* invalid. Returns `charts: undefined` when nothing valid remains so the UI
-* falls back to rendering no chart (matching the opt-in default).
-*/
-function validateCharts(params) {
-	const { charts, columnDefs, evalId } = params;
-	if (!charts || charts.length === 0) return {
-		charts: void 0,
-		warnings: []
-	};
-	const columnsByKey = new Map(columnDefs.map((def) => [def.key, def]));
-	const warnings = [];
-	const sanitized = [];
-	for (const chart of charts) {
-		const result = sanitizeChart(chart, columnsByKey, evalId, warnings);
-		if (result) sanitized.push(result);
-	}
-	return {
-		charts: sanitized.length > 0 ? sanitized : void 0,
-		warnings
-	};
-}
-//#endregion
 //#region ../runner/src/columnBuilder.ts
 /**
 * Normalize a user-provided score definition (either a function or an
@@ -2513,98 +2437,6 @@ async function loadConfig() {
 	}
 }
 //#endregion
-//#region ../runner/src/discovery.ts
-const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
-const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
-function parseEvalMetas(filePath, content) {
-	const metas = [];
-	let searchIndex = 0;
-	while (searchIndex < content.length) {
-		const defineEvalIndex = content.indexOf("defineEval", searchIndex);
-		if (defineEvalIndex === -1) break;
-		const extracted = extractDefineEvalObject(content, defineEvalIndex);
-		if (!extracted) {
-			searchIndex = defineEvalIndex + 10;
-			continue;
-		}
-		const id = evalIdMatchRegex.exec(extracted.objectText)?.[1];
-		if (id !== void 0) {
-			const result = {
-				filePath,
-				id
-			};
-			const title = evalTitleMatchRegex.exec(extracted.objectText)?.[1];
-			if (title !== void 0) result.title = title;
-			metas.push(result);
-		}
-		searchIndex = extracted.nextIndex;
-	}
-	return metas;
-}
-function extractDefineEvalObject(content, defineEvalIndex) {
-	const openParenIndex = content.indexOf("(", defineEvalIndex);
-	if (openParenIndex === -1) return void 0;
-	const objectStartIndex = content.indexOf("{", openParenIndex);
-	if (objectStartIndex === -1) return void 0;
-	let depth = 0;
-	let quote;
-	let inBlockComment = false;
-	let inLineComment = false;
-	let isEscaped = false;
-	for (let index = objectStartIndex; index < content.length; index++) {
-		const currentChar = content[index];
-		const nextChar = content[index + 1];
-		if (inLineComment) {
-			if (currentChar === "\n") inLineComment = false;
-			continue;
-		}
-		if (inBlockComment) {
-			if (currentChar === "*" && nextChar === "/") {
-				inBlockComment = false;
-				index++;
-			}
-			continue;
-		}
-		if (quote) {
-			if (isEscaped) {
-				isEscaped = false;
-				continue;
-			}
-			if (currentChar === "\\") {
-				isEscaped = true;
-				continue;
-			}
-			if (currentChar === quote) quote = void 0;
-			continue;
-		}
-		if (currentChar === "/" && nextChar === "/") {
-			inLineComment = true;
-			index++;
-			continue;
-		}
-		if (currentChar === "/" && nextChar === "*") {
-			inBlockComment = true;
-			index++;
-			continue;
-		}
-		if (currentChar === "\"" || currentChar === "'" || currentChar === "`") {
-			quote = currentChar;
-			continue;
-		}
-		if (currentChar === "{") {
-			depth++;
-			continue;
-		}
-		if (currentChar === "}") {
-			depth--;
-			if (depth === 0) return {
-				nextIndex: index + 1,
-				objectText: content.slice(objectStartIndex, index + 1)
-			};
-		}
-	}
-}
-//#endregion
 //#region ../runner/src/evalModuleLoader.ts
 /**
 * Import one eval module with a cache key derived from its current source so
@@ -2692,30 +2524,6 @@ function setLatestRunInfoMap(params) {
 	for (const evalId of evalIds) latestRunInfoMap.set(evalId, info);
 }
 //#endregion
-//#region ../runner/src/gitState.ts
-function runGitCommand(workspaceRoot, args) {
-	const result = spawnSync("git", args, {
-		cwd: workspaceRoot,
-		encoding: "utf8",
-		stdio: [
-			"ignore",
-			"pipe",
-			"ignore"
-		]
-	});
-	return {
-		status: result.status,
-		stdout: result.stdout.trim()
-	};
-}
-/** Read the current git commit for the workspace, if available. */
-function readGitWorktreeState(workspaceRoot) {
-	const insideWorktree = runGitCommand(workspaceRoot, ["rev-parse", "--is-inside-work-tree"]);
-	if (insideWorktree.status !== 0 || insideWorktree.stdout !== "true") return { commitSha: null };
-	const commitResult = runGitCommand(workspaceRoot, ["rev-parse", "HEAD"]);
-	return { commitSha: commitResult.status === 0 ? commitResult.stdout : null };
-}
-//#endregion
 //#region ../runner/src/outputArtifacts.ts
 const mimeTypeExtensionMap = {
 	"application/json": ".json",
@@ -2867,6 +2675,235 @@ async function recomputeEvalStatusesInRuns(params) {
 	return updatedRuns;
 }
 //#endregion
+//#region ../runner/src/runPersistence.ts
+const SHORT_ID_PATTERN = /^r(\d+)$/;
+/**
+* Generate a filesystem-safe, sortable run id combining a UTC timestamp
+* with a short random suffix.
+*/
+function generateRunId() {
+	const now = /* @__PURE__ */ new Date();
+	const pad = (n) => String(n).padStart(2, "0");
+	return `${`${String(now.getUTCFullYear())}-${pad(now.getUTCMonth() + 1)}-${pad(now.getUTCDate())}T${pad(now.getUTCHours())}-${pad(now.getUTCMinutes())}-${pad(now.getUTCSeconds())}Z`}_${Math.random().toString(36).slice(2, 8)}`;
+}
+function parseShortIdNum(shortId) {
+	if (shortId === void 0) return null;
+	const match = SHORT_ID_PATTERN.exec(shortId);
+	if (!match) return null;
+	const num = Number(match[1]);
+	if (!Number.isFinite(num)) return null;
+	return num;
+}
+/**
+* Return the next `shortId` number to assign based on the existing
+* loaded snapshots. Legacy runs that don't match the `r\d+` format are
+* ignored.
+*/
+function nextShortIdFromSnapshots(snapshots) {
+	let maxNum = -1;
+	for (const snapshot of snapshots) {
+		const num = parseShortIdNum(snapshot.manifest.shortId);
+		if (num !== null && num > maxNum) maxNum = num;
+	}
+	return maxNum + 1;
+}
+async function loadPersistedRunSnapshots(localStateDir) {
+	const runsDir = join(localStateDir, "runs");
+	const entriesResult = await resultify(() => readdir(runsDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	const snapshots = [];
+	const runDirs = entriesResult.value.filter((entry) => entry.isDirectory()).map((entry) => join(runsDir, entry.name)).toSorted();
+	for (const runDir of runDirs) {
+		const snapshot = await loadPersistedRunSnapshot(runDir);
+		if (!snapshot) continue;
+		snapshots.push(snapshot);
+	}
+	return snapshots;
+}
+async function persistCaseDetail(runDir, caseDetail) {
+	await writeFile(join(runDir, "case-details", `${encodeCaseDetailFileName(caseDetail.caseId)}.json`), JSON.stringify(caseDetail, null, 2));
+}
+function getLastRunStatuses(params) {
+	const latestRunInfos = getLatestRunInfos(params);
+	return new Map([...latestRunInfos].map(([evalId, info]) => [evalId, info.status]));
+}
+/**
+* Return the latest scoped run metadata for each eval based on persisted and
+* in-memory runs.
+*/
+function getLatestRunInfos(params) {
+	const { runs, knownEvals } = params;
+	const knownEvalMetas = [...knownEvals];
+	const manualScoreKeysByEval = new Map(knownEvalMetas.map((evalMeta) => [evalMeta.id, evalMeta.columnDefs.filter((columnDef) => columnDef.isManualScore === true).map((columnDef) => columnDef.key)]));
+	const orderedRuns = [...runs].toSorted((a, b) => new Date(getRunFreshnessTimestamp(a.manifest)).getTime() - new Date(getRunFreshnessTimestamp(b.manifest)).getTime());
+	const latestRunInfos = /* @__PURE__ */ new Map();
+	for (const run of orderedRuns) for (const evalId of getRunEvalIds(run, knownEvalMetas.map((evalMeta) => evalMeta.id))) latestRunInfos.set(evalId, {
+		status: getEvalStatusForRun(run, evalId, manualScoreKeysByEval.get(evalId) ?? []),
+		startedAt: getRunFreshnessTimestamp(run.manifest),
+		commitSha: run.manifest.commitSha ?? null,
+		evalSourceFingerprint: run.manifest.evalSourceFingerprints[evalId] ?? null
+	});
+	return latestRunInfos;
+}
+function toLastRunStatus$1(status) {
+	return status === "pending" ? null : status;
+}
+async function loadPersistedRunSnapshot(runDir) {
+	const manifest = await readParsedJsonFile(join(runDir, "run.json"), { safeParse: runManifestSchema.safeParse.bind(runManifestSchema) });
+	if (!manifest) return null;
+	const summary = await readParsedJsonFile(join(runDir, "summary.json"), { safeParse: runSummarySchema.safeParse.bind(runSummarySchema) });
+	if (!summary) return null;
+	return {
+		runDir,
+		manifest,
+		summary,
+		cases: await readCaseRows(runDir),
+		caseDetails: await readCaseDetails(runDir)
+	};
+}
+async function readParsedJsonFile(filePath, schema) {
+	const fileResult = await resultify(() => readFile(filePath, "utf-8"));
+	if (fileResult.error) return null;
+	const jsonResult = resultify(() => JSON.parse(fileResult.value));
+	if (jsonResult.error) return null;
+	const parsed = schema.safeParse(jsonResult.value);
+	if (!parsed.success) return null;
+	return parsed.data;
+}
+async function readCaseRows(runDir) {
+	const fileResult = await resultify(() => readFile(join(runDir, "cases.jsonl"), "utf-8"));
+	if (fileResult.error) return [];
+	const rows = [];
+	for (const rawLine of fileResult.value.split("\n")) {
+		const line = rawLine.trim();
+		if (line.length === 0) continue;
+		const jsonResult = resultify(() => JSON.parse(line));
+		if (jsonResult.error) continue;
+		const parsed = caseRowSchema.safeParse(jsonResult.value);
+		if (!parsed.success) continue;
+		rows.push(parsed.data);
+	}
+	return rows;
+}
+async function readCaseDetails(runDir) {
+	const detailsDir = join(runDir, "case-details");
+	const entriesResult = await resultify(() => readdir(detailsDir, { withFileTypes: true }));
+	if (entriesResult.error) return /* @__PURE__ */ new Map();
+	const caseDetails = /* @__PURE__ */ new Map();
+	for (const entry of entriesResult.value) {
+		if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
+		const detail = await readParsedJsonFile(join(detailsDir, entry.name), { safeParse: caseDetailSchema.safeParse.bind(caseDetailSchema) });
+		if (!detail) continue;
+		caseDetails.set(detail.caseId, detail);
+	}
+	return caseDetails;
+}
+function getRunEvalIds(run, knownEvalIds) {
+	const evalIds = new Set(run.cases.map((caseRow) => caseRow.evalId));
+	if (run.manifest.target.mode === "evalIds") for (const evalId of run.manifest.target.evalIds ?? []) evalIds.add(evalId);
+	else if (run.manifest.target.mode === "all" && evalIds.size === 0) for (const evalId of knownEvalIds) evalIds.add(evalId);
+	return [...evalIds];
+}
+function getEvalStatusForRun(run, evalId, manualScoreKeys) {
+	const evalCases = run.cases.filter((caseRow) => caseRow.evalId === evalId);
+	if (evalCases.length > 0) {
+		if (hasPendingManualScores(evalCases, manualScoreKeys)) return "unscored";
+		return toLastRunStatus$1(deriveStatusFromCaseRows({ caseRows: evalCases }));
+	}
+	return toLastRunStatus$1(deriveStatusFromChildStatuses({
+		statuses: [],
+		lifecycleStatus: run.manifest.status
+	}));
+}
+function hasPendingManualScores(caseRows, manualScoreKeys) {
+	if (manualScoreKeys.length === 0) return false;
+	return caseRows.some((caseRow) => manualScoreKeys.some((key) => {
+		const value = caseRow.columns[key];
+		return typeof value !== "number" || !Number.isFinite(value);
+	}));
+}
+function encodeCaseDetailFileName(caseId) {
+	return encodeURIComponent(caseId);
+}
+//#endregion
+//#region ../runner/src/moduleIsolation.ts
+const isolationParam = "agent-evals-isolate";
+const pathSegmentSeparatorPattern = /[\\/]+/;
+const isolationStorage = new AsyncLocalStorage();
+const activeIsolationRoots = /* @__PURE__ */ new Map();
+let hooksRegistered = false;
+const requireFromRunner = createRequire(import.meta.url);
+const agentPackageUrlBySpecifier = new Map([
+	"@ls-stack/agent-eval",
+	"@agent-evals/sdk",
+	"@agent-evals/shared",
+	"@agent-evals/runner",
+	"@agent-evals/runner/run-child"
+].flatMap((specifier) => {
+	try {
+		return [[specifier, pathToFileURL(requireFromRunner.resolve(specifier)).href]];
+	} catch {
+		return [];
+	}
+}));
+function isAgentEvalsPackageSpecifier(specifier) {
+	return specifier === "@ls-stack/agent-eval" || specifier === "@agent-evals/sdk" || specifier === "@agent-evals/shared" || specifier === "@agent-evals/runner" || specifier.startsWith("@ls-stack/agent-eval/") || specifier.startsWith("@agent-evals/sdk/") || specifier.startsWith("@agent-evals/shared/") || specifier.startsWith("@agent-evals/runner/");
+}
+function getIsolationKeyFromParent(parentURL) {
+	if (!parentURL?.startsWith("file:")) return null;
+	const value = new URL(parentURL).searchParams.get(isolationParam);
+	return activeIsolationRoots.has(value ?? "") ? value : null;
+}
+function isWorkspaceFile(url, workspaceRoot) {
+	if (url.protocol !== "file:") return false;
+	const relativePath = relative(workspaceRoot, fileURLToPath(url));
+	if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) return false;
+	const segments = relativePath.split(pathSegmentSeparatorPattern);
+	return !segments.includes("node_modules") && !segments.includes(".agent-evals");
+}
+function addIsolationParam(url, key) {
+	const moduleUrl = new URL(url);
+	if (moduleUrl.searchParams.get(isolationParam) === key) return url;
+	moduleUrl.searchParams.set(isolationParam, key);
+	return moduleUrl.href;
+}
+function registerModuleIsolationHooks() {
+	if (hooksRegistered) return;
+	hooksRegistered = true;
+	registerHooks({ resolve(specifier, context, nextResolve) {
+		const agentPackageUrl = agentPackageUrlBySpecifier.get(specifier);
+		if (agentPackageUrl !== void 0) return {
+			url: agentPackageUrl,
+			shortCircuit: true
+		};
+		const resolved = nextResolve(specifier, context);
+		if (isAgentEvalsPackageSpecifier(specifier)) return resolved;
+		const activeContext = isolationStorage.getStore();
+		const inferredKey = getIsolationKeyFromParent(context.parentURL);
+		const isolationKey = activeContext?.key ?? inferredKey;
+		if (isolationKey === null) return resolved;
+		const workspaceRoot = activeContext?.workspaceRoot ?? activeIsolationRoots.get(isolationKey);
+		if (workspaceRoot === void 0) return resolved;
+		if (!isWorkspaceFile(new URL(resolved.url), workspaceRoot)) return resolved;
+		return {
+			...resolved,
+			url: addIsolationParam(resolved.url, isolationKey)
+		};
+	} });
+}
+/**
+* Execute module loading and eval code with fresh workspace module URLs.
+*
+* Node does not expose an ESM cache reset API, so the runner appends a
+* run-scoped query parameter to workspace file imports. Package imports are
+* left alone so SDK singletons, such as the eval registry, remain shared.
+*/
+async function runWithModuleIsolation(context, fn) {
+	registerModuleIsolationHooks();
+	activeIsolationRoots.set(context.key, context.workspaceRoot);
+	return await isolationStorage.run(context, fn);
+}
+//#endregion
 //#region ../runner/src/traceDisplay.ts
 function isRecord$1(value) {
 	return typeof value === "object" && value !== null;
@@ -2954,12 +2991,16 @@ async function callWithUnknownResult(fn, args) {
 	return await Reflect.apply(fn, void 0, args);
 }
 async function runCase(params) {
-	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, signal, startTime, cacheAdapter, cacheMode, codeFingerprint, artifactDir, runId } = params;
+	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, startTime, cacheAdapter, cacheMode, codeFingerprint, moduleIsolation, artifactDir, runId } = params;
 	const { scope, error: executeError } = await runInEvalScope(evalCase.id, async () => {
-		await Reflect.apply(evalDef.execute, evalDef, [{
-			input: evalCase.input,
-			signal
-		}]);
+		const execute = async () => {
+			await Reflect.apply(evalDef.execute, evalDef, [{ input: evalCase.input }]);
+		};
+		if (moduleIsolation === void 0) {
+			await execute();
+			return;
+		}
+		await runWithModuleIsolation(moduleIsolation, execute);
 	}, {
 		input: evalCase.input,
 		cacheContext: cacheAdapter ? {
@@ -2996,11 +3037,15 @@ async function runCase(params) {
 	const scoringTraces = {};
 	if (!nonAssertError && scope.assertionFailures.length === 0 && evalDef.scores) for (const [key, def] of Object.entries(evalDef.scores)) {
 		const { compute, passThreshold, label } = normalizeScoreDef(def);
-		const scoreRun = await runInEvalScope(evalCase.id, async () => await callWithUnknownResult(compute, [{
-			input: evalCase.input,
-			outputs: { ...scope.outputs },
-			case: evalCase
-		}]), {
+		const scoreRun = await runInEvalScope(evalCase.id, async () => {
+			const computeScore = async () => await callWithUnknownResult(compute, [{
+				input: evalCase.input,
+				outputs: { ...scope.outputs },
+				case: evalCase
+			}]);
+			if (moduleIsolation === void 0) return await computeScore();
+			return await runWithModuleIsolation(moduleIsolation, computeScore);
+		}, {
 			input: evalCase.input,
 			cacheContext: cacheAdapter ? {
 				adapter: cacheAdapter,
@@ -3119,172 +3164,20 @@ function toAssertionFailure(message, error = void 0) {
 	} : { message };
 }
 //#endregion
-//#region ../runner/src/runPersistence.ts
-const SHORT_ID_PATTERN = /^r(\d+)$/;
-/**
-* Generate a filesystem-safe, sortable run id combining a UTC timestamp
-* with a short random suffix.
-*/
-function generateRunId() {
-	const now = /* @__PURE__ */ new Date();
-	const pad = (n) => String(n).padStart(2, "0");
-	return `${`${String(now.getUTCFullYear())}-${pad(now.getUTCMonth() + 1)}-${pad(now.getUTCDate())}T${pad(now.getUTCHours())}-${pad(now.getUTCMinutes())}-${pad(now.getUTCSeconds())}Z`}_${Math.random().toString(36).slice(2, 8)}`;
-}
-function parseShortIdNum(shortId) {
-	if (shortId === void 0) return null;
-	const match = SHORT_ID_PATTERN.exec(shortId);
-	if (!match) return null;
-	const num = Number(match[1]);
-	if (!Number.isFinite(num)) return null;
-	return num;
-}
-/**
-* Return the next `shortId` number to assign based on the existing
-* loaded snapshots. Legacy runs that don't match the `r\d+` format are
-* ignored.
-*/
-function nextShortIdFromSnapshots(snapshots) {
-	let maxNum = -1;
-	for (const snapshot of snapshots) {
-		const num = parseShortIdNum(snapshot.manifest.shortId);
-		if (num !== null && num > maxNum) maxNum = num;
-	}
-	return maxNum + 1;
-}
-async function loadPersistedRunSnapshots(localStateDir) {
-	const runsDir = join(localStateDir, "runs");
-	const entriesResult = await resultify(() => readdir(runsDir, { withFileTypes: true }));
-	if (entriesResult.error) return [];
-	const snapshots = [];
-	const runDirs = entriesResult.value.filter((entry) => entry.isDirectory()).map((entry) => join(runsDir, entry.name)).toSorted();
-	for (const runDir of runDirs) {
-		const snapshot = await loadPersistedRunSnapshot(runDir);
-		if (!snapshot) continue;
-		snapshots.push(snapshot);
-	}
-	return snapshots;
-}
-async function persistCaseDetail(runDir, caseDetail) {
-	await writeFile(join(runDir, "case-details", `${encodeCaseDetailFileName(caseDetail.caseId)}.json`), JSON.stringify(caseDetail, null, 2));
-}
-function getLastRunStatuses(params) {
-	const latestRunInfos = getLatestRunInfos(params);
-	return new Map([...latestRunInfos].map(([evalId, info]) => [evalId, info.status]));
-}
-/**
-* Return the latest scoped run metadata for each eval based on persisted and
-* in-memory runs.
-*/
-function getLatestRunInfos(params) {
-	const { runs, knownEvals } = params;
-	const knownEvalMetas = [...knownEvals];
-	const manualScoreKeysByEval = new Map(knownEvalMetas.map((evalMeta) => [evalMeta.id, evalMeta.columnDefs.filter((columnDef) => columnDef.isManualScore === true).map((columnDef) => columnDef.key)]));
-	const orderedRuns = [...runs].toSorted((a, b) => new Date(getRunFreshnessTimestamp(a.manifest)).getTime() - new Date(getRunFreshnessTimestamp(b.manifest)).getTime());
-	const latestRunInfos = /* @__PURE__ */ new Map();
-	for (const run of orderedRuns) for (const evalId of getRunEvalIds(run, knownEvalMetas.map((evalMeta) => evalMeta.id))) latestRunInfos.set(evalId, {
-		status: getEvalStatusForRun(run, evalId, manualScoreKeysByEval.get(evalId) ?? []),
-		startedAt: getRunFreshnessTimestamp(run.manifest),
-		commitSha: run.manifest.commitSha ?? null,
-		evalSourceFingerprint: run.manifest.evalSourceFingerprints[evalId] ?? null
-	});
-	return latestRunInfos;
-}
-function toLastRunStatus$1(status) {
-	return status === "pending" ? null : status;
-}
-async function loadPersistedRunSnapshot(runDir) {
-	const manifest = await readParsedJsonFile(join(runDir, "run.json"), { safeParse: runManifestSchema.safeParse.bind(runManifestSchema) });
-	if (!manifest) return null;
-	const summary = await readParsedJsonFile(join(runDir, "summary.json"), { safeParse: runSummarySchema.safeParse.bind(runSummarySchema) });
-	if (!summary) return null;
-	return {
-		runDir,
-		manifest,
-		summary,
-		cases: await readCaseRows(runDir),
-		caseDetails: await readCaseDetails(runDir)
-	};
-}
-async function readParsedJsonFile(filePath, schema) {
-	const fileResult = await resultify(() => readFile(filePath, "utf-8"));
-	if (fileResult.error) return null;
-	const jsonResult = resultify(() => JSON.parse(fileResult.value));
-	if (jsonResult.error) return null;
-	const parsed = schema.safeParse(jsonResult.value);
-	if (!parsed.success) return null;
-	return parsed.data;
-}
-async function readCaseRows(runDir) {
-	const fileResult = await resultify(() => readFile(join(runDir, "cases.jsonl"), "utf-8"));
-	if (fileResult.error) return [];
-	const rows = [];
-	for (const rawLine of fileResult.value.split("\n")) {
-		const line = rawLine.trim();
-		if (line.length === 0) continue;
-		const jsonResult = resultify(() => JSON.parse(line));
-		if (jsonResult.error) continue;
-		const parsed = caseRowSchema.safeParse(jsonResult.value);
-		if (!parsed.success) continue;
-		rows.push(parsed.data);
-	}
-	return rows;
-}
-async function readCaseDetails(runDir) {
-	const detailsDir = join(runDir, "case-details");
-	const entriesResult = await resultify(() => readdir(detailsDir, { withFileTypes: true }));
-	if (entriesResult.error) return /* @__PURE__ */ new Map();
-	const caseDetails = /* @__PURE__ */ new Map();
-	for (const entry of entriesResult.value) {
-		if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
-		const detail = await readParsedJsonFile(join(detailsDir, entry.name), { safeParse: caseDetailSchema.safeParse.bind(caseDetailSchema) });
-		if (!detail) continue;
-		caseDetails.set(detail.caseId, detail);
-	}
-	return caseDetails;
-}
-function getRunEvalIds(run, knownEvalIds) {
-	const evalIds = new Set(run.cases.map((caseRow) => caseRow.evalId));
-	if (run.manifest.target.mode === "evalIds") for (const evalId of run.manifest.target.evalIds ?? []) evalIds.add(evalId);
-	else if (run.manifest.target.mode === "all" && evalIds.size === 0) for (const evalId of knownEvalIds) evalIds.add(evalId);
-	return [...evalIds];
-}
-function getEvalStatusForRun(run, evalId, manualScoreKeys) {
-	const evalCases = run.cases.filter((caseRow) => caseRow.evalId === evalId);
-	if (evalCases.length > 0) {
-		if (hasPendingManualScores(evalCases, manualScoreKeys)) return "unscored";
-		return toLastRunStatus$1(deriveStatusFromCaseRows({ caseRows: evalCases }));
-	}
-	return toLastRunStatus$1(deriveStatusFromChildStatuses({
-		statuses: [],
-		lifecycleStatus: run.manifest.status
-	}));
-}
-function hasPendingManualScores(caseRows, manualScoreKeys) {
-	if (manualScoreKeys.length === 0) return false;
-	return caseRows.some((caseRow) => manualScoreKeys.some((key) => {
-		const value = caseRow.columns[key];
-		return typeof value !== "number" || !Number.isFinite(value);
-	}));
-}
-function encodeCaseDetailFileName(caseId) {
-	return encodeURIComponent(caseId);
-}
-//#endregion
 //#region ../runner/src/runQueue.ts
 async function executeQueuedCases(params) {
-	const { runState, queuedCases, concurrency, globalTraceDisplay } = params;
+	const { queuedCases, concurrency, globalTraceDisplay } = params;
 	let nextCaseIndex = 0;
 	let workerError = void 0;
 	const workerCount = Math.min(concurrency, queuedCases.length);
 	const workers = Array.from({ length: workerCount }, async () => {
-		while (!runState.abortController.signal.aborted && workerError === void 0) {
+		while (workerError === void 0) {
 			const queuedCase = queuedCases[nextCaseIndex];
 			nextCaseIndex += 1;
 			if (queuedCase === void 0) return;
 			try {
 				await executeQueuedCase({
 					queuedCase,
-					runState,
 					globalTraceDisplay
 				});
 			} catch (error) {
@@ -3298,11 +3191,10 @@ async function executeQueuedCases(params) {
 	if (workerError !== void 0) throw new Error(typeof workerError === "string" ? workerError : typeof workerError === "number" || typeof workerError === "boolean" || typeof workerError === "bigint" ? String(workerError) : workerError === null ? "null" : "Unknown queue worker error");
 }
 async function executeQueuedCase(params) {
-	const { queuedCase, runState, globalTraceDisplay } = params;
+	const { queuedCase, globalTraceDisplay } = params;
 	const startTime = Date.now();
 	const result = await queuedCase.execute({
 		globalTraceDisplay,
-		signal: runState.abortController.signal,
 		startTime
 	});
 	await queuedCase.onComplete(result);
@@ -3353,7 +3245,48 @@ function pickWinningTrial(params) {
 	if (medianAttempt === void 0) throw new Error("Expected at least one trial attempt");
 	return medianAttempt;
 }
-async function executeRun({ runState, request, runDir, config, evals, cacheStore, lastRunStatusMap, latestRunInfoMap, emitEvent, emitDiscoveryEvent, getSourceFingerprint, getConfiguredConcurrency, getSortedEvalMetas, getTargetEvals }) {
+async function finalizePreparedCase(params) {
+	const { runState, runDir, preparedEval, preparedCase, onCaseFinished, emitEvent } = params;
+	if (preparedCase.finalized || preparedCase.trialResults.length === 0) return;
+	preparedCase.finalized = true;
+	const winningTrial = pickWinningTrial({
+		strategy: runState.manifest.trialSelection,
+		attempts: preparedCase.trialResults,
+		scoreKeys: preparedEval.scoreKeys
+	});
+	if (winningTrial.bufferedCacheStore !== null) await winningTrial.bufferedCacheStore.commit();
+	runState.cases.push(winningTrial.caseRow);
+	runState.caseDetails.set(preparedCase.caseId, winningTrial.caseDetail);
+	preparedEval.mergeColumns(winningTrial.caseDetail.columns);
+	if (winningTrial.caseRow.status === "pass") runState.summary.passedCases++;
+	else if (winningTrial.caseRow.status === "error") runState.summary.errorCases++;
+	else runState.summary.failedCases++;
+	await writeFile(join(runDir, "traces", `${preparedCase.caseId}.json`), JSON.stringify(winningTrial.caseDetail.trace, null, 2));
+	await persistCaseDetail(runDir, winningTrial.caseDetail);
+	onCaseFinished?.(winningTrial.caseDetail, winningTrial.caseRow);
+	emitEvent(runState, {
+		type: "case.finished",
+		runId: runState.manifest.id,
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		payload: winningTrial.caseRow
+	});
+	preparedEval.evalCaseRows.push(winningTrial.caseRow);
+}
+function getPreparedCaseOrderKey(caseRow) {
+	return `${caseRow.evalId}\u0000${caseRow.caseId}`;
+}
+function sortCaseRowsByPreparedOrder(caseRows, preparedEvals) {
+	const orderByCase = /* @__PURE__ */ new Map();
+	let order = 0;
+	for (const preparedEval of preparedEvals) for (const preparedCase of preparedEval.preparedCases) {
+		orderByCase.set(`${preparedEval.evalMeta.id}\u0000${preparedCase.caseId}`, order);
+		order++;
+	}
+	caseRows.sort((left, right) => {
+		return (orderByCase.get(getPreparedCaseOrderKey(left)) ?? Number.MAX_SAFE_INTEGER) - (orderByCase.get(getPreparedCaseOrderKey(right)) ?? Number.MAX_SAFE_INTEGER);
+	});
+}
+async function executeRun({ runState, request, runDir, config, evals, cacheStore, lastRunStatusMap, latestRunInfoMap, emitEvent, emitDiscoveryEvent, workspaceRoot, getSourceFingerprint, getConfiguredConcurrency, getSortedEvalMetas, getTargetEvals, onCaseFinished }) {
 	try {
 		const targetEvals = getTargetEvals(request);
 		emitEvent(runState, {
@@ -3362,14 +3295,16 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
 			payload: runState.manifest
 		});
-		const allCaseRows = [];
 		const evalErrors = [];
 		const queuedCases = [];
 		const preparedEvals = [];
 		const cacheMode = runState.manifest.cacheMode ?? "use";
 		const cacheEnabled = config.cache?.enabled !== false;
+		const moduleIsolation = {
+			key: runState.manifest.id,
+			workspaceRoot
+		};
 		for (const evalMeta of targetEvals) {
-			if (runState.abortController.signal.aborted) break;
 			const evalFilePath = evalMeta.sourceFilePath;
 			let codeFingerprint = "";
 			try {
@@ -3381,7 +3316,9 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			else delete runState.manifest.evalSourceFingerprints[evalMeta.id];
 			try {
 				const registry = getEvalRegistry();
-				await loadEvalModule(evalFilePath, codeFingerprint);
+				await runWithModuleIsolation(moduleIsolation, async () => {
+					await loadEvalModule(evalFilePath, codeFingerprint);
+				});
 				const entry = registry.get(evalMeta.id);
 				if (!entry) {
 					evalErrors.push({
@@ -3390,74 +3327,87 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 					});
 					continue;
 				}
-				await entry.use(async (evalDef) => {
-					const cases = filterEvalCases(resolveRunnableEvalCases({
-						cases: typeof evalDef.cases === "function" ? await evalDef.cases() : evalDef.cases ?? [],
-						evalId: evalMeta.id
-					}), request.target.evalIds, request.target.caseIds, evalMeta.id);
-					runState.summary.totalCases += cases.length;
-					const accumulatedColumns = /* @__PURE__ */ new Map();
-					const evalCaseRows = [];
-					const preparedCases = [];
-					const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
-					const manualScoreKeys = Object.freeze(Object.keys(evalDef.manualScores ?? {}));
-					preparedEvals.push({
-						evalMeta,
-						accumulatedColumns,
-						evalCaseRows,
-						preparedCases,
-						scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
-						mergeColumns: (columns) => {
-							mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
-						}
-					});
-					for (const evalCase of cases) {
-						if (runState.abortController.signal.aborted) break;
-						const trialResults = [];
-						preparedCases.push({
-							caseId: evalCase.id,
-							trialResults
-						});
-						for (let trial = 0; trial < request.trials; trial++) {
-							const bufferedCacheStore = cacheEnabled && cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
-							queuedCases.push({
-								execute: async ({ startTime, signal, globalTraceDisplay }) => {
-									const { caseDetail, caseRowUpdate } = await runCase({
-										evalDef,
-										evalId: evalMeta.id,
-										evalCase,
-										globalTraceDisplay,
-										trial,
-										signal,
-										startTime,
-										cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
-										cacheMode,
-										codeFingerprint,
-										artifactDir: join(runDir, "artifacts"),
-										runId: runState.manifest.id
-									});
-									return {
-										caseDetail,
-										caseRow: {
-											caseId: evalCase.id,
+				await runWithModuleIsolation(moduleIsolation, async () => {
+					await entry.use(async (evalDef) => {
+						const cases = filterEvalCases(resolveRunnableEvalCases({
+							cases: typeof evalDef.cases === "function" ? await evalDef.cases() : evalDef.cases ?? [],
+							evalId: evalMeta.id
+						}), request.target.evalIds, request.target.caseIds, evalMeta.id);
+						runState.summary.totalCases += cases.length;
+						const accumulatedColumns = /* @__PURE__ */ new Map();
+						const evalCaseRows = [];
+						const preparedCases = [];
+						const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
+						const manualScoreKeys = Object.freeze(Object.keys(evalDef.manualScores ?? {}));
+						const preparedEval = {
+							evalMeta,
+							accumulatedColumns,
+							evalCaseRows,
+							preparedCases,
+							scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
+							mergeColumns: (columns) => {
+								mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
+							}
+						};
+						preparedEvals.push(preparedEval);
+						for (const evalCase of cases) {
+							const trialResults = [];
+							const preparedCase = {
+								caseId: evalCase.id,
+								trialResults,
+								finalized: false
+							};
+							preparedCases.push(preparedCase);
+							for (let trial = 0; trial < request.trials; trial++) {
+								const bufferedCacheStore = cacheEnabled && cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
+								queuedCases.push({
+									execute: async ({ startTime, globalTraceDisplay }) => {
+										const { caseDetail, caseRowUpdate } = await runCase({
+											evalDef,
 											evalId: evalMeta.id,
-											status: caseRowUpdate.status ?? "pending",
-											latencyMs: caseRowUpdate.latencyMs ?? null,
-											columns: caseRowUpdate.columns ?? {},
-											trial
-										}
-									};
-								},
-								onComplete: ({ caseDetail, caseRow }) => {
-									trialResults.push({
-										caseDetail,
-										caseRow,
-										bufferedCacheStore
-									});
-								}
-							});
+											evalCase,
+											globalTraceDisplay,
+											trial,
+											startTime,
+											cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
+											cacheMode,
+											codeFingerprint,
+											moduleIsolation,
+											artifactDir: join(runDir, "artifacts"),
+											runId: runState.manifest.id
+										});
+										return {
+											caseDetail,
+											caseRow: {
+												caseId: evalCase.id,
+												evalId: evalMeta.id,
+												status: caseRowUpdate.status ?? "pending",
+												latencyMs: caseRowUpdate.latencyMs ?? null,
+												columns: caseRowUpdate.columns ?? {},
+												trial
+											}
+										};
+									},
+									onComplete: async ({ caseDetail, caseRow }) => {
+										trialResults.push({
+											caseDetail,
+											caseRow,
+											bufferedCacheStore
+										});
+										if (trialResults.length !== request.trials) return;
+										await finalizePreparedCase({
+											runState,
+											runDir,
+											preparedEval,
+											preparedCase,
+											onCaseFinished,
+											emitEvent
+										});
+									}
+								});
+							}
 						}
-					}
+					});
 				});
 			} catch (error) {
 				console.error(`Error running eval ${evalMeta.id}:`, error);
@@ -3475,37 +3425,19 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			}
 		}
 		await executeQueuedCases({
-			runState,
 			queuedCases,
 			concurrency: getConfiguredConcurrency(),
 			globalTraceDisplay: config.traceDisplay
 		});
 		for (const preparedEval of preparedEvals) {
-			for (const preparedCase of preparedEval.preparedCases) {
-				if (preparedCase.trialResults.length === 0) continue;
-				const winningTrial = pickWinningTrial({
-					strategy: runState.manifest.trialSelection,
-					attempts: preparedCase.trialResults,
-					scoreKeys: preparedEval.scoreKeys
-				});
-				if (winningTrial.bufferedCacheStore !== null) await winningTrial.bufferedCacheStore.commit();
-				runState.cases.push(winningTrial.caseRow);
-				runState.caseDetails.set(preparedCase.caseId, winningTrial.caseDetail);
-				preparedEval.mergeColumns(winningTrial.caseDetail.columns);
-				if (winningTrial.caseRow.status === "pass") runState.summary.passedCases++;
-				else if (winningTrial.caseRow.status === "error") runState.summary.errorCases++;
-				else runState.summary.failedCases++;
-				await writeFile(join(runDir, "traces", `${preparedCase.caseId}.json`), JSON.stringify(winningTrial.caseDetail.trace, null, 2));
-				await persistCaseDetail(runDir, winningTrial.caseDetail);
-				emitEvent(runState, {
-					type: "case.finished",
-					runId: runState.manifest.id,
-					timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-					payload: winningTrial.caseRow
-				});
-				preparedEval.evalCaseRows.push(winningTrial.caseRow);
-				allCaseRows.push(winningTrial.caseRow);
-			}
+			for (const preparedCase of preparedEval.preparedCases) await finalizePreparedCase({
+				runState,
+				runDir,
+				preparedEval,
+				preparedCase,
+				onCaseFinished,
+				emitEvent
+			});
 			preparedEval.evalMeta.columnDefs = [...preparedEval.accumulatedColumns.values()];
 			lastRunStatusMap.set(preparedEval.evalMeta.id, toLastRunStatus(deriveStatusFromCaseRows({ caseRows: preparedEval.evalCaseRows })));
 			const latestStatus = lastRunStatusMap.get(preparedEval.evalMeta.id) ?? null;
@@ -3516,9 +3448,11 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 				evalSourceFingerprint: runState.manifest.evalSourceFingerprints[preparedEval.evalMeta.id] ?? null
 			});
 		}
+		sortCaseRowsByPreparedOrder(runState.cases, preparedEvals);
+		for (const preparedEval of preparedEvals) sortCaseRowsByPreparedOrder(preparedEval.evalCaseRows, preparedEvals);
 		const endTime = /* @__PURE__ */ new Date();
 		runState.summary.totalDurationMs = endTime.getTime() - new Date(runState.manifest.startedAt).getTime();
-		const finalStatus = runState.abortController.signal.aborted ? "cancelled" : evalErrors.length > 0 ? "error" : "completed";
+		const finalStatus = evalErrors.length > 0 ? "error" : "completed";
 		runState.summary.status = finalStatus;
 		runState.manifest.status = finalStatus;
 		const completedRunAt = endTime.toISOString();
@@ -3540,6 +3474,7 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 				evalSourceFingerprint: runState.manifest.evalSourceFingerprints[evalId] ?? null
 			});
 		}
+		await persistRunState(runState);
 		emitEvent(runState, {
 			type: "run.summary",
 			runId: runState.manifest.id,
@@ -3558,7 +3493,6 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
 			payload: runState.summary
 		});
-		await persistRunState(runState);
 		emitDiscoveryEvent();
 	} catch (error) {
 		const message = error instanceof Error ? error.message : String(error);
@@ -3566,13 +3500,13 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 		runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
 		runState.summary.status = "error";
 		runState.summary.errorMessage = message;
+		await persistRunState(runState);
 		emitEvent(runState, {
 			type: "run.error",
 			runId: runState.manifest.id,
 			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
 			payload: { message }
 		});
-		await persistRunState(runState);
 		emitDiscoveryEvent();
 	}
 }
@@ -3580,896 +3514,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-//#region ../runner/src/runner.ts
-const globMagicCharacters = new Set([
-	"*",
-	"?",
-	"[",
-	"]",
-	"{",
-	"}",
-	"(",
-	")",
-	"!",
-	"+",
-	"@"
-]);
-function hasGlobMagic(value) {
-	for (const char of value) if (globMagicCharacters.has(char)) return true;
-	return false;
-}
-function getWatchRootForIncludePattern(params) {
-	const segments = params.pattern.replaceAll("\\", "/").split("/").filter((part) => part !== "");
-	const firstGlobSegmentIndex = segments.findIndex(hasGlobMagic);
-	if (firstGlobSegmentIndex === -1) return dirname(resolve(params.workspaceRoot, params.pattern));
-	if (firstGlobSegmentIndex === 0) return params.workspaceRoot;
-	return resolve(params.workspaceRoot, segments.slice(0, firstGlobSegmentIndex).join("/"));
-}
-function getWatchRootsForIncludePatterns(params) {
-	const roots = /* @__PURE__ */ new Set();
-	for (const pattern of params.patterns) roots.add(getWatchRootForIncludePattern({
-		pattern,
-		workspaceRoot: params.workspaceRoot
-	}));
-	if (roots.size === 0) return [params.workspaceRoot];
-	return [...roots];
-}
-/** Create an in-memory eval runner bound to the current workspace config. */
-function createRunner({ watchForChanges = true } = {}) {
-	let config;
-	let workspaceRoot;
-	let localStateDir;
-	let cacheStore;
-	const evals = /* @__PURE__ */ new Map();
-	const runs = /* @__PURE__ */ new Map();
-	const lastRunStatusMap = /* @__PURE__ */ new Map();
-	const latestRunInfoMap = /* @__PURE__ */ new Map();
-	const discoveryListeners = /* @__PURE__ */ new Set();
-	let nextShortIdNum = 0;
-	let discoveryWatcher;
-	let discoveryRefreshTimer;
-	function toWorkspaceRelativePath(filePath) {
-		return relative(workspaceRoot, filePath).replaceAll("\\", "/");
-	}
-	function getSortedEvalMetas() {
-		return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
-	}
-	function getSourceFingerprint(source) {
-		return createHash("sha256").update(source).digest("hex");
-	}
-	function getConfiguredConcurrency() {
-		const configuredConcurrency = config.concurrency;
-		if (typeof configuredConcurrency !== "number" || !Number.isFinite(configuredConcurrency)) return 1;
-		return Math.max(1, Math.floor(configuredConcurrency));
-	}
-	const runner = {
-		async init() {
-			config = await loadConfig();
-			workspaceRoot = config.workspaceRoot ?? process.cwd();
-			localStateDir = resolve(workspaceRoot, ".agent-evals");
-			await mkdir(localStateDir, { recursive: true });
-			await mkdir(join(localStateDir, "runs"), { recursive: true });
-			cacheStore = createFsCacheStore({
-				workspaceRoot,
-				dir: config.cache?.dir,
-				maxEntriesPerEval: config.cache?.maxEntriesPerEval
-			});
-			await loadPersistedRuns();
-			await runner.refreshDiscovery();
-			if (watchForChanges) await setupWatcher();
-		},
-		async listCache() {
-			return cacheStore.list();
-		},
-		async clearCache(filter) {
-			await cacheStore.clear(filter);
-		},
-		async recomputeStatusesForEval(evalId) {
-			const evalMeta = evals.get(evalId);
-			if (!evalMeta) return { updatedRuns: 0 };
-			const registry = getEvalRegistry();
-			await loadEvalModule(evalMeta.sourceFilePath, evalMeta.sourceFingerprint ?? void 0);
-			const entry = registry.get(evalId);
-			if (!entry) return { updatedRuns: 0 };
-			const scoreThresholds = /* @__PURE__ */ new Map();
-			entry.use((evalDef) => {
-				for (const [key, def] of Object.entries(evalDef.scores ?? {})) {
-					const threshold = normalizeScoreDef(def).passThreshold;
-					if (threshold !== void 0) scoreThresholds.set(key, threshold);
-				}
-				for (const [key, def] of Object.entries(evalDef.manualScores ?? {})) if (def.passThreshold !== void 0) scoreThresholds.set(key, def.passThreshold);
-			});
-			const updatedRuns = await recomputeEvalStatusesInRuns({
-				runs: runs.values(),
-				evalId,
-				evalExists: evals.has(evalId),
-				scoreThresholds,
-				persistCaseDetail
-			});
-			emitDiscoveryEvent();
-			return { updatedRuns };
-		},
-		async cleanRunsForEval(evalId) {
-			let deletedRuns = 0;
-			for (const [runId, run] of [...runs]) {
-				if (!runTouchesEval({
-					target: run.manifest.target,
-					caseRows: run.cases,
-					evalId,
-					evalExists: evals.has(evalId)
-				})) continue;
-				if (run.manifest.status === "running") continue;
-				runs.delete(runId);
-				await rm(run.runDir, {
-					recursive: true,
-					force: true
-				});
-				deletedRuns += 1;
-			}
-			emitDiscoveryEvent();
-			return { deletedRuns };
-		},
-		async updateManualScore({ runId, caseId, scoreKey, value }) {
-			const run = runs.get(runId);
-			if (!run) return {
-				updated: false,
-				reason: "Run not found"
-			};
-			if (run.manifest.status === "running") return {
-				updated: false,
-				reason: "Run is still running"
-			};
-			const caseRow = run.cases.find((row) => row.caseId === caseId);
-			if (!caseRow) return {
-				updated: false,
-				reason: "Case not found"
-			};
-			const evalMeta = evals.get(caseRow.evalId);
-			if (!evalMeta) return {
-				updated: false,
-				reason: "Eval not found"
-			};
-			if (evalMeta.columnDefs.find((def) => def.key === scoreKey)?.isManualScore !== true) return {
-				updated: false,
-				reason: "Manual score not found"
-			};
-			const caseDetail = run.caseDetails.get(caseId);
-			if (!caseDetail) return {
-				updated: false,
-				reason: "Case detail not found"
-			};
-			caseRow.columns[scoreKey] = value;
-			caseDetail.columns[scoreKey] = value;
-			const scoreThresholds = /* @__PURE__ */ new Map();
-			for (const def of evalMeta.columnDefs) {
-				if (def.isScore !== true || def.passThreshold === void 0) continue;
-				scoreThresholds.set(def.key, def.passThreshold);
-			}
-			const nextStatus = recomputePersistedCaseStatus(caseRow, caseDetail, scoreThresholds);
-			caseRow.status = nextStatus;
-			caseDetail.status = nextStatus;
-			const derivedSummary = deriveScopedSummaryFromCases({ caseRows: run.cases });
-			run.summary.totalCases = derivedSummary.totalCases;
-			run.summary.passedCases = derivedSummary.passedCases;
-			run.summary.failedCases = derivedSummary.failedCases;
-			run.summary.errorCases = derivedSummary.errorCases;
-			run.summary.cancelledCases = derivedSummary.cancelledCases;
-			run.summary.totalDurationMs = derivedSummary.totalDurationMs;
-			await persistCaseDetail(run.runDir, caseDetail);
-			await persistRunState(run);
-			emitDiscoveryEvent();
-			return {
-				updated: true,
-				run: {
-					manifest: run.manifest,
-					summary: run.summary,
-					cases: run.cases
-				},
-				caseDetail
-			};
-		},
-		async deleteRun(runId) {
-			const run = runs.get(runId);
-			if (!run) return { deleted: false };
-			if (run.manifest.status === "running") return { deleted: false };
-			runs.delete(runId);
-			await rm(run.runDir, {
-				recursive: true,
-				force: true
-			});
-			emitDiscoveryEvent();
-			return { deleted: true };
-		},
-		getEvals() {
-			const gitState = readGitWorktreeState(workspaceRoot);
-			const result = [];
-			for (const meta of getSortedEvalMetas()) result.push(buildEvalSummary({
-				meta,
-				config,
-				gitState,
-				latestRun: latestRunInfoMap.get(meta.id),
-				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
-			}));
-			return result;
-		},
-		getEval(id) {
-			const meta = evals.get(id);
-			if (!meta) return void 0;
-			return buildEvalSummary({
-				meta,
-				config,
-				gitState: readGitWorktreeState(workspaceRoot),
-				latestRun: latestRunInfoMap.get(meta.id),
-				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
-			});
-		},
-		async refreshDiscovery() {
-			const patterns = config.include;
-			const discovered = [];
-			for (const pattern of patterns) {
-				const files = await glob(pattern, {
-					cwd: workspaceRoot,
-					absolute: true
-				});
-				discovered.push(...files);
-			}
-			evals.clear();
-			for (const filePath of discovered) try {
-				const content = await readFile(filePath, "utf-8");
-				const discoveredMetas = parseEvalMetas(filePath, content);
-				const sourceFingerprint = getSourceFingerprint(content);
-				const registry = getEvalRegistry();
-				try {
-					await loadEvalModule(filePath, sourceFingerprint);
-				} catch {}
-				for (const meta of discoveredMetas) {
-					const discoveredEntry = registry.get(meta.id);
-					const title = meta.title;
-					let columnDefs = buildDeclaredColumnDefs(void 0, void 0, void 0);
-					let stats;
-					let charts;
-					discoveredEntry?.use((evalDef) => {
-						columnDefs = buildDeclaredColumnDefs(evalDef.columns, evalDef.scores, evalDef.manualScores);
-						stats = evalDef.stats;
-						const validated = validateCharts({
-							charts: evalDef.charts,
-							columnDefs,
-							evalId: meta.id
-						});
-						for (const warning of validated.warnings) console.warn(warning);
-						charts = validated.charts;
-					});
-					evals.set(meta.id, {
-						id: meta.id,
-						title,
-						filePath: toWorkspaceRelativePath(meta.filePath),
-						sourceFilePath: meta.filePath,
-						sourceFingerprint,
-						columnDefs,
-						caseCount: null,
-						stats,
-						charts
-					});
-				}
-			} catch {}
-			emitDiscoveryEvent();
-		},
-		async startRun(request) {
-			const runId = generateRunId();
-			const shortId = `r${String(nextShortIdNum++)}`;
-			const now = (/* @__PURE__ */ new Date()).toISOString();
-			const cacheMode = request.cache?.mode ?? "use";
-			const runDir = join(localStateDir, "runs", runId);
-			const manifest = {
-				id: runId,
-				shortId,
-				status: "running",
-				startedAt: now,
-				endedAt: null,
-				commitSha: readGitWorktreeState(workspaceRoot).commitSha,
-				evalSourceFingerprints: {},
-				target: request.target,
-				trials: request.trials,
-				trialSelection: config.trialSelection ?? "lowestScore",
-				cacheMode
-			};
-			const summary = {
-				runId,
-				status: "running",
-				totalCases: 0,
-				passedCases: 0,
-				failedCases: 0,
-				errorCases: 0,
-				cancelledCases: 0,
-				totalDurationMs: null,
-				errorMessage: null
-			};
-			const abortController = new AbortController();
-			const runState = {
-				runDir,
-				manifest,
-				summary,
-				cases: [],
-				caseDetails: /* @__PURE__ */ new Map(),
-				listeners: /* @__PURE__ */ new Set(),
-				abortController
-			};
-			runs.set(runId, runState);
-			setLatestRunInfoMap({
-				latestRunInfoMap,
-				evalIds: getTargetEvalIds({
-					request,
-					sortedEvalIds: getSortedEvalMetas().map((meta) => meta.id),
-					knownEvalIds: new Set(evals.keys())
-				}),
-				info: {
-					status: "running",
-					startedAt: now,
-					commitSha: manifest.commitSha ?? null,
-					evalSourceFingerprint: null
-				}
-			});
-			await mkdir(runDir, { recursive: true });
-			await mkdir(join(runDir, "traces"), { recursive: true });
-			await mkdir(join(runDir, "artifacts"), { recursive: true });
-			await mkdir(join(runDir, "case-details"), { recursive: true });
-			await writeFile(join(runDir, "run.json"), JSON.stringify(manifest, null, 2));
-			executeRun({
-				runState,
-				request,
-				runDir,
-				config,
-				evals,
-				cacheStore,
-				lastRunStatusMap,
-				latestRunInfoMap,
-				emitEvent,
-				emitDiscoveryEvent,
-				getSourceFingerprint,
-				getConfiguredConcurrency,
-				getSortedEvalMetas,
-				getTargetEvals
-			});
-			return {
-				manifest,
-				summary,
-				cases: []
-			};
-		},
-		getRuns() {
-			return [...runs.values()].map((r) => r.manifest);
-		},
-		getRun(id) {
-			const run = runs.get(id);
-			if (!run) return void 0;
-			return {
-				manifest: run.manifest,
-				summary: run.summary,
-				cases: run.cases
-			};
-		},
-		cancelRun(id) {
-			const run = runs.get(id);
-			if (!run) return;
-			run.abortController.abort();
-			run.manifest.status = "cancelled";
-			run.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
-			run.summary.status = "cancelled";
-			emitEvent(run, {
-				type: "run.cancelled",
-				runId: id,
-				timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-				payload: run.summary
-			});
-		},
-		getCaseDetail(runId, caseId) {
-			const run = runs.get(runId);
-			if (!run) return void 0;
-			return run.caseDetails.get(caseId);
-		},
-		subscribe(runId, listener) {
-			const run = runs.get(runId);
-			if (!run) return () => {};
-			run.listeners.add(listener);
-			return () => {
-				run.listeners.delete(listener);
-			};
-		},
-		subscribeDiscovery(listener) {
-			discoveryListeners.add(listener);
-			return () => {
-				discoveryListeners.delete(listener);
-			};
-		},
-		async close() {
-			if (discoveryRefreshTimer !== void 0) {
-				clearTimeout(discoveryRefreshTimer);
-				discoveryRefreshTimer = void 0;
-			}
-			const watcher = discoveryWatcher;
-			if (watcher === void 0) return;
-			discoveryWatcher = void 0;
-			await watcher.close();
-		},
-		getWorkspaceRoot() {
-			return workspaceRoot;
-		},
-		getArtifactPath(artifactId_) {
-			return resolveArtifactPath(join(localStateDir, "runs"), artifactId_);
-		}
-	};
-	async function setupWatcher() {
-		const watcher = watch(getWatchRootsForIncludePatterns({
-			patterns: config.include,
-			workspaceRoot
-		}), {
-			ignoreInitial: true,
-			persistent: true
-		});
-		discoveryWatcher = watcher;
-		const scheduleRefresh = () => {
-			if (discoveryRefreshTimer !== void 0) clearTimeout(discoveryRefreshTimer);
-			discoveryRefreshTimer = setTimeout(() => {
-				discoveryRefreshTimer = void 0;
-				runner.refreshDiscovery();
-			}, 50);
-		};
-		watcher.on("change", scheduleRefresh);
-		watcher.on("add", scheduleRefresh);
-		watcher.on("unlink", scheduleRefresh);
-		watcher.on("addDir", scheduleRefresh);
-		watcher.on("unlinkDir", scheduleRefresh);
-		await new Promise((ready) => {
-			watcher.once("ready", ready);
-		});
-	}
-	function emitDiscoveryEvent() {
-		const lastRunStatuses = getLastRunStatuses({
-			runs: runs.values(),
-			knownEvals: evals.values()
-		});
-		const latestRunInfos = getLatestRunInfos({
-			runs: runs.values(),
-			knownEvals: evals.values()
-		});
-		lastRunStatusMap.clear();
-		for (const [evalId, status] of lastRunStatuses) lastRunStatusMap.set(evalId, status);
-		latestRunInfoMap.clear();
-		for (const [evalId, info] of latestRunInfos) latestRunInfoMap.set(evalId, info);
-		const event = {
-			type: "discovery.updated",
-			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-			payload: runner.getEvals()
-		};
-		for (const listener of discoveryListeners) listener(event);
-	}
-	function getTargetEvals(request) {
-		if (request.target.evalIds && request.target.evalIds.length > 0) return request.target.evalIds.map((id) => evals.get(id)).filter((e) => e !== void 0);
-		return getSortedEvalMetas();
-	}
-	function emitEvent(runState, event) {
-		for (const listener of runState.listeners) try {
-			listener(event);
-		} catch {}
-	}
-	async function loadPersistedRuns() {
-		runs.clear();
-		const persistedRuns = await loadPersistedRunSnapshots(localStateDir);
-		nextShortIdNum = nextShortIdFromSnapshots(persistedRuns);
-		for (const persistedRun of persistedRuns) runs.set(persistedRun.manifest.id, {
-			...persistedRun,
-			listeners: /* @__PURE__ */ new Set(),
-			abortController: new AbortController()
-		});
-	}
-	return runner;
-}
-//#endregion
-//#region src/cli.ts
-function parseArgs(argv) {
-	const normalizedArgv = argv.filter((arg) => arg !== "--no-env");
-	const args = {
-		command: "help",
-		subcommand: void 0,
-		showHelp: false,
-		helpTopic: "global",
-		unknownHelpTarget: void 0,
-		evalIds: [],
-		caseIds: [],
-		trials: 1,
-		json: false,
-		port: 4100,
-		cacheMode: "use",
-		clearCache: false,
-		all: false,
-		loadEnv: normalizedArgv.length === argv.length
-	};
-	const command = normalizedArgv[0];
-	if (command === "--help" || command === "-h") {
-		args.showHelp = true;
-		return args;
-	}
-	if (isCliCommand(command)) {
-		args.command = command;
-		args.helpTopic = command === "help" ? "global" : command;
-	} else if (command !== void 0 && !command.startsWith("-")) args.unknownHelpTarget = command;
-	let cursor = 1;
-	if (args.command === "cache") {
-		const sub = normalizedArgv[cursor];
-		if (sub === "list" || sub === "clear") {
-			args.subcommand = sub;
-			args.helpTopic = `cache ${sub}`;
-			cursor++;
-		} else if (sub !== void 0 && !sub.startsWith("-")) args.unknownHelpTarget = `cache ${sub}`;
-	}
-	for (let i = cursor; i < normalizedArgv.length; i++) {
-		const arg = normalizedArgv[i];
-		const next = normalizedArgv[i + 1];
-		if (arg === "--help" || arg === "-h") args.showHelp = true;
-		else if (arg === "--eval" && next) {
-			args.evalIds.push(...next.split(","));
-			i++;
-		} else if (arg === "--case" && next) {
-			args.caseIds.push(...next.split(","));
-			i++;
-		} else if (arg === "--trials" && next) {
-			args.trials = Number(next);
-			i++;
-		} else if (arg === "--json") args.json = true;
-		else if (arg === "--port" && next) {
-			args.port = Number(next);
-			i++;
-		} else if (arg === "--cache" && next) {
-			if (next === "use" || next === "bypass" || next === "refresh") args.cacheMode = next;
-			i++;
-		} else if (arg === "--no-cache") args.cacheMode = "bypass";
-		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
-		else if (arg === "--clear-cache") args.clearCache = true;
-		else if (arg === "--all") args.all = true;
-	}
-	return args;
-}
-/**
-* Run the Agent Evals CLI against the current workspace.
-*
-* @param argv Raw command-line arguments excluding the executable name.
-*/
-async function runCli(argv) {
-	const args = parseArgs(argv);
-	if (args.loadEnv && !loadWorkspaceEnv()) {
-		process.exit(1);
-		return;
-	}
-	if (args.showHelp) {
-		if (args.unknownHelpTarget !== void 0) {
-			console.error(`No help found for "${args.unknownHelpTarget}".`);
-			process.exit(1);
-			return;
-		}
-		printHelp(args.helpTopic);
-		return;
-	}
-	switch (args.command) {
-		case "app":
-			await commandApp(args);
-			break;
-		case "list":
-			await commandList(args);
-			break;
-		case "run":
-			await commandRun(args);
-			break;
-		case "cache":
-			await commandCache(args);
-			break;
-		default:
-			printHelp(args.helpTopic);
-			break;
-	}
-}
-function isCliCommand(command) {
-	return command === "app" || command === "list" || command === "run" || command === "cache" || command === "help";
-}
-function loadWorkspaceEnv() {
-	const envPath = resolve(process.cwd(), ".env");
-	if (!existsSync(envPath)) return true;
-	const loadResult = resultify(() => {
-		process.loadEnvFile(envPath);
-	});
-	if (loadResult.error) {
-		console.error(`Failed to load .env at ${envPath}: ${loadResult.error.message}`);
-		return false;
-	}
-	return true;
-}
-const currentDir = dirname(fileURLToPath(import.meta.url));
-const repoRoot = resolve(currentDir, "../../..");
-const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
-function hasRepoWebWorkspace() {
-	return existsSync(resolve(repoRoot, "apps/web/package.json"));
-}
-async function ensureWebUiIsBuilt() {
-	if (!hasRepoWebWorkspace()) return;
-	console.info("Preparing web UI...");
-	await new Promise((resolvePromise, rejectPromise) => {
-		const child = spawn(pnpmCommand, [
-			"--filter",
-			"@agent-evals/web",
-			"build"
-		], {
-			cwd: repoRoot,
-			stdio: "inherit"
-		});
-		child.once("error", (error) => {
-			rejectPromise(error);
-		});
-		child.once("exit", (code, signal) => {
-			if (signal) {
-				rejectPromise(/* @__PURE__ */ new Error(`Web UI build stopped with signal ${signal}.`));
-				return;
-			}
-			if (code !== 0) {
-				rejectPromise(/* @__PURE__ */ new Error(`Web UI build failed with exit code ${String(code)}.`));
-				return;
-			}
-			resolvePromise();
-		});
-	});
-}
-function isHonoAppModule(mod) {
-	if (typeof mod !== "object" || mod === null || !("app" in mod)) return false;
-	const { app } = mod;
-	return typeof app === "object" && app !== null && "fetch" in app && typeof app.fetch === "function";
-}
-function isServerRunnerModule(mod) {
-	if (typeof mod !== "object" || mod === null || !("initRunner" in mod)) return false;
-	return typeof mod.initRunner === "function";
-}
-async function commandApp(args) {
-	await ensureWebUiIsBuilt();
-	const { serve } = await import("@hono/node-server");
-	const bundledWebDist = resolve(currentDir, "apps/web/dist");
-	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-7qDBq_ub.mjs");
-	const runnerModule = await import("./runner-uzzY8kk1.mjs");
-	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
-	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
-	await runnerModule.initRunner();
-	console.info(`Agent Evals app: http://localhost:${String(args.port)}`);
-	serve({
-		fetch: appModule.app.fetch,
-		port: args.port
-	});
-}
-async function commandList(args_) {
-	const runner = createRunner({ watchForChanges: false });
-	await runner.init();
-	const evals = runner.getEvals();
-	if (evals.length === 0) {
-		console.info("No eval files found.");
-		return;
-	}
-	console.info("Discovered evals:\n");
-	for (const ev of evals) {
-		const displayStatus = getEvalDisplayStatus({
-			freshnessStatus: ev.freshnessStatus,
-			stale: ev.stale,
-			outdated: ev.outdated,
-			lastRunStatus: ev.lastRunStatus
-		});
-		const title = getEvalTitle(ev);
-		console.info(`  ${title}`);
-		console.info(`    id: ${ev.id}`);
-		console.info(`    file: ${ev.filePath}`);
-		if (displayStatus !== "pending") console.info(`    status: ${displayStatus}`);
-		if (ev.caseCount !== null) console.info(`    cases: ${String(ev.caseCount)}`);
-		console.info("");
-	}
-}
-async function commandRun(args) {
-	const runner = createRunner({ watchForChanges: false });
-	await runner.init();
-	if (args.clearCache) {
-		await runner.clearCache();
-		if (!args.json) {
-			console.info("Cleared cache before run.");
-			console.info("");
-		}
-	}
-	const target = args.caseIds.length > 0 ? {
-		mode: "caseIds",
-		caseIds: args.caseIds,
-		evalIds: args.evalIds.length > 0 ? args.evalIds : void 0
-	} : args.evalIds.length > 0 ? {
-		mode: "evalIds",
-		evalIds: args.evalIds
-	} : { mode: "all" };
-	const run = await runner.startRun({
-		target,
-		trials: args.trials,
-		cache: { mode: args.cacheMode }
-	});
-	if (!args.json) {
-		console.info(`Run started: ${run.manifest.id}`);
-		console.info(`Trials: ${String(args.trials)}`);
-		if (args.cacheMode !== "use") console.info(`Cache mode: ${args.cacheMode}`);
-		console.info("");
-	}
-	await waitForRunCompletion(runner, run.manifest.id);
-	const finalRun = runner.getRun(run.manifest.id);
-	if (!finalRun) {
-		process.exit(1);
-		return;
-	}
-	const { summary } = finalRun;
-	if (args.json) console.info(JSON.stringify(summary, null, 2));
-	else {
-		console.info("--- Run Summary ---");
-		console.info(`Status: ${summary.status}`);
-		console.info(`Total: ${String(summary.totalCases)}`);
-		console.info(`Passed: ${String(summary.passedCases)}`);
-		console.info(`Failed: ${String(summary.failedCases)}`);
-		console.info(`Errors: ${String(summary.errorCases)}`);
-		if (summary.totalCases > 0) console.info(`Pass Rate: ${String(summary.passedCases)}/${String(summary.totalCases)}`);
-		if (summary.totalDurationMs !== null) console.info(`Duration: ${(summary.totalDurationMs / 1e3).toFixed(1)}s`);
-	}
-	if (summary.failedCases > 0 || summary.errorCases > 0) process.exit(1);
-}
-async function commandCache(args) {
-	const runner = createRunner({ watchForChanges: false });
-	await runner.init();
-	if (args.subcommand === "list" || args.subcommand === void 0) {
-		const entries = await runner.listCache();
-		if (args.json) {
-			console.info(JSON.stringify(entries, null, 2));
-			return;
-		}
-		if (entries.length === 0) {
-			console.info("No cache entries.");
-			return;
-		}
-		console.info(`Cache entries (${String(entries.length)}):\n`);
-		for (const entry of entries) {
-			console.info(`  ${entry.namespace}`);
-			console.info(`    key: ${entry.key}`);
-			const operationLabel = entry.operationType === "span" ? `${entry.operationName} (span ${entry.spanKind ?? "unknown"})` : `${entry.operationName} (value)`;
-			console.info(`    operation: ${operationLabel}`);
-			console.info(`    stored: ${entry.storedAt}`);
-			console.info(`    size: ${String(entry.sizeBytes)} bytes`);
-			console.info("");
-		}
-		return;
-	}
-	if (args.subcommand === "clear") {
-		if (args.evalIds.length > 0) {
-			for (const evalId of args.evalIds) {
-				const entries = await runner.listCache();
-				const prefix = `${evalId}__`;
-				const matching = entries.filter((entry) => entry.namespace.startsWith(prefix));
-				for (const entry of matching) await runner.clearCache({
-					namespace: entry.namespace,
-					key: entry.key
-				});
-			}
-			console.info(`Cleared cache entries for: ${args.evalIds.join(", ")}`);
-			return;
-		}
-		if (args.all) {
-			await runner.clearCache();
-			console.info("Cleared all cache entries.");
-			return;
-		}
-		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
-		process.exit(1);
-		return;
-	}
-	printHelp(args.helpTopic);
-}
-async function waitForRunCompletion(runner, runId) {
-	return new Promise((resolvePromise) => {
-		const check = () => {
-			const run = runner.getRun(runId);
-			if (!run || run.manifest.status === "completed" || run.manifest.status === "cancelled" || run.manifest.status === "error") {
-				resolvePromise();
-				return;
-			}
-			setTimeout(check, 200);
-		};
-		check();
-	});
-}
-function printHelp(topic = "global") {
-	if (topic === "app") {
-		console.info(`
-agent-evals app - Start server with UI
-
-Usage:
-  agent-evals app [flags]
-
-Flags:
-  --port <n>                 Server port (default: 4100)
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "list") {
-		console.info(`
-agent-evals list - List discovered evals
-
-Usage:
-  agent-evals list [flags]
-
-Flags:
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "run") {
-		console.info(`
-agent-evals run - Run evals
-
-Usage:
-  agent-evals run [flags]
-
-Flags:
-  --eval <id>                Run specific eval(s) (comma-separated)
-  --case <id>                Run specific case(s) (comma-separated)
-  --trials <n>               Number of trials per case
-  --json                     Output run summary as JSON
-  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
-  --no-cache                 Shortcut for --cache bypass
-  --refresh-cache            Shortcut for --cache refresh
-  --clear-cache              Clear the cache before starting the run
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
-		console.info(`
-agent-evals cache - Manage cached operation entries
-
-Usage:
-  agent-evals cache list [flags]
-  agent-evals cache clear --eval <id>
-  agent-evals cache clear --all
-
-Flags:
-  --eval <id>                Clear entries for specific eval(s) (comma-separated)
-  --all                      Confirm clearing every cached entry
-  --json                     Output cache listing as JSON
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show this help
-  `);
-		return;
-	}
-	console.info(`
-agent-evals - LLM/Agent eval runner
-
-Commands:
-  app                        Start server with UI
-  list                       List discovered evals
-  run                        Run evals
-  cache list                 List cached operation entries
-  cache clear --eval <id>    Clear cache entries for one eval
-  cache clear --all          Clear every cached entry
-  help                       Show this help
-
-Options:
-  --eval <id>                Run specific eval(s) (comma-separated)
-  --case <id>                Run specific case(s) (comma-separated)
-  --trials <n>               Number of trials per case
-  --json                     Output results as JSON
-  --port <n>                 Server port (default: 4100)
-  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
-  --no-cache                 Shortcut for --cache bypass
-  --refresh-cache            Shortcut for --cache refresh
-  --clear-cache              Clear the cache before starting the run
-  --no-env                   Disable automatic .env loading
-  --help, -h                 Show help
-  `);
-}
-//#endregion
-export { columnDefSchema as $, evalStatsConfigSchema as A, evalChartTypeSchema as B, spanCacheOptionsSchema as C, setEvalOutput as Ct, evalFreshnessStatusSchema as D, getEvalRegistry as Dt, caseRowSchema as E, defineEval as Et, evalChartBuiltinMetricSchema as F, traceAttributeDisplaySchema as G, traceAttributeDisplayFormatSchema as H, evalChartColorSchema as I, traceSpanErrorSchema as J, traceDisplayConfigSchema as K, evalChartConfigSchema as L, scoreTraceSchema as M, evalChartAggregateSchema as N, evalStatAggregateSchema as O, evalChartAxisSchema as P, cellValueSchema as Q, evalChartMetricSchema as R, serializedCacheSpanSchema as S, runInEvalScope as St, caseDetailSchema as T, repoFile as Tt, traceAttributeDisplayInputSchema as U, evalChartsConfigSchema as V, traceAttributeDisplayPlacementSchema as W, traceSpanSchema as X, traceSpanKindSchema as Y, traceSpanWarningSchema as Z, cacheListItemSchema as _, getCurrentScope as _t, sseEnvelopeSchema as a, repoFileRefSchema as at, cacheRecordingOpSchema as b, isInEvalScope as bt, deriveScopedSummaryFromCases as c, buildTraceTree as ct, runManifestSchema as d, evalTracer as dt, columnFormatSchema as et, runSummarySchema as f, hashCacheKey as ft, cacheFileSchema as g, evalAssert as gt, cacheEntrySchema as h, appendToEvalOutput as ht, updateManualScoreRequestSchema as i, numberDisplayOptionsSchema as it, evalSummarySchema as j, evalStatItemSchema as k, deriveStatusFromCaseRows as l, captureEvalSpanError as lt, trialSelectionModeSchema as m, EvalAssertionError as mt, createRunner as n, fileRefSchema as nt, getEvalTitle as o, runArtifactRefSchema as ot, agentEvalsConfigSchema as p, hashCacheKeySync as pt, traceDisplayInputConfigSchema as q, createRunRequestSchema as r, jsonCellSchema as rt, getEvalDisplayStatus as s, z$1 as st, runCli as t, columnKindSchema as tt, deriveStatusFromChildStatuses as u, evalSpan as ut, cacheModeSchema as v, getEvalCaseInput as vt, assertionFailureSchema as w, setScopeCacheContext as wt, cacheRecordingSchema as x, mergeEvalOutput as xt, cacheOperationTypeSchema as y, incrementEvalOutput as yt, evalChartTooltipExtraSchema as z };
+export { evalChartAxisSchema as $, runManifestSchema as A, evalTracer as At, cacheRecordingSchema as B, mergeEvalOutput as Bt, updateManualScoreRequestSchema as C, numberDisplayOptionsSchema as Ct, deriveScopedSummaryFromCases as D, buildTraceTree as Dt, getEvalDisplayStatus as E, z$1 as Et, cacheFileSchema as F, evalAssert as Ft, caseRowSchema as G, defineEval as Gt, spanCacheOptionsSchema as H, setEvalOutput as Ht, cacheListItemSchema as I, getCurrentScope as It, evalStatItemSchema as J, evalFreshnessStatusSchema as K, getEvalRegistry as Kt, cacheModeSchema as L, getEvalCaseInput as Lt, agentEvalsConfigSchema as M, hashCacheKeySync as Mt, trialSelectionModeSchema as N, EvalAssertionError as Nt, deriveStatusFromCaseRows as O, captureEvalSpanError as Ot, cacheEntrySchema as P, appendToEvalOutput as Pt, evalChartAggregateSchema as Q, cacheOperationTypeSchema as R, incrementEvalOutput as Rt, createRunRequestSchema as S, jsonCellSchema as St, getEvalTitle as T, runArtifactRefSchema as Tt, assertionFailureSchema as U, setScopeCacheContext as Ut, serializedCacheSpanSchema as V, runInEvalScope as Vt, caseDetailSchema as W, repoFile as Wt, evalSummarySchema as X, evalStatsConfigSchema as Y, scoreTraceSchema as Z, loadEvalModule as _, cellValueSchema as _t, loadPersistedRunSnapshot as a, evalChartTypeSchema as at, normalizeScoreDef as b, columnKindSchema as bt, persistCaseDetail as c, traceAttributeDisplayInputSchema as ct, recomputePersistedCaseStatus as d, traceDisplayConfigSchema as dt, evalChartBuiltinMetricSchema as et, runTouchesEval as f, traceDisplayInputConfigSchema as ft, setLatestRunInfoMap as g, traceSpanWarningSchema as gt, getTargetEvalIds as h, traceSpanSchema as ht, getLatestRunInfos as i, evalChartTooltipExtraSchema as it, runSummarySchema as j, hashCacheKey as jt, deriveStatusFromChildStatuses as k, evalSpan as kt, persistRunState as l, traceAttributeDisplayPlacementSchema as lt, buildEvalSummary as m, traceSpanKindSchema as mt, generateRunId as n, evalChartConfigSchema as nt, loadPersistedRunSnapshots as o, evalChartsConfigSchema as ot, resolveArtifactPath as p, traceSpanErrorSchema as pt, evalStatAggregateSchema as q, getLastRunStatuses as r, evalChartMetricSchema as rt, nextShortIdFromSnapshots as s, traceAttributeDisplayFormatSchema as st, executeRun as t, evalChartColorSchema as tt, recomputeEvalStatusesInRuns as u, traceAttributeDisplaySchema as ut, loadConfig as v, columnDefSchema as vt, sseEnvelopeSchema as w, repoFileRefSchema as wt, createFsCacheStore as x, fileRefSchema as xt, buildDeclaredColumnDefs as y, columnFormatSchema as yt, cacheRecordingOpSchema as z, isInEvalScope as zt };