@ls-stack/agent-eval 0.5.0 → 0.7.0

package/dist/cli-BTtgQLjB.mjs

@@ -0,0 +1,1285 @@
+import { D as deriveScopedSummaryFromCases, E as getEvalDisplayStatus, Kt as getEvalRegistry, T as getEvalTitle, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, j as runSummarySchema, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-HaMahl6b.mjs";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join, relative, resolve } from "node:path";
+import { watch } from "chokidar";
+import { glob } from "glob";
+import { existsSync } from "node:fs";
+import { resultify } from "t-result";
+import { fileURLToPath } from "node:url";
+import { spawn, spawnSync } from "node:child_process";
+//#region ../runner/src/chartValidation.ts
+function isValidColumnMetric(metric, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(metric.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart metric references unknown column "${metric.key}" — dropped`);
+		return false;
+	}
+	if (metric.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart metric "${metric.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function isValidTooltipExtra(extra, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(extra.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart tooltip extra references unknown column "${extra.key}" — dropped`);
+		return false;
+	}
+	if (extra.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart tooltip extra "${extra.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function sanitizeChart(chart, columnsByKey, evalId, warnings) {
+	const metrics = chart.metrics.filter((metric) => {
+		if (metric.source === "builtin") return true;
+		return isValidColumnMetric(metric, columnsByKey, evalId, warnings);
+	});
+	if (metrics.length === 0) {
+		warnings.push(`[${evalId}] chart had no valid metrics after validation — chart dropped`);
+		return null;
+	}
+	const tooltipExtras = chart.tooltipExtras?.filter((extra) => {
+		if (extra.source === "builtin") return true;
+		return isValidTooltipExtra(extra, columnsByKey, evalId, warnings);
+	});
+	return {
+		...chart,
+		metrics,
+		tooltipExtras: tooltipExtras?.length ? tooltipExtras : void 0
+	};
+}
+/**
+* Validate and sanitize an authored `charts` config against the eval's
+* declared columns. Drops metrics/extras that reference unknown columns or
+* misuse `passThresholdRate`, and drops entire charts whose metrics are all
+* invalid. Returns `charts: undefined` when nothing valid remains so the UI
+* falls back to rendering no chart (matching the opt-in default).
+*/
+function validateCharts(params) {
+	const { charts, columnDefs, evalId } = params;
+	if (!charts || charts.length === 0) return {
+		charts: void 0,
+		warnings: []
+	};
+	const columnsByKey = new Map(columnDefs.map((def) => [def.key, def]));
+	const warnings = [];
+	const sanitized = [];
+	for (const chart of charts) {
+		const result = sanitizeChart(chart, columnsByKey, evalId, warnings);
+		if (result) sanitized.push(result);
+	}
+	return {
+		charts: sanitized.length > 0 ? sanitized : void 0,
+		warnings
+	};
+}
+//#endregion
+//#region ../runner/src/discovery.ts
+const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
+const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
+function parseEvalMetas(filePath, content) {
+	const metas = [];
+	let searchIndex = 0;
+	while (searchIndex < content.length) {
+		const defineEvalIndex = content.indexOf("defineEval", searchIndex);
+		if (defineEvalIndex === -1) break;
+		const extracted = extractDefineEvalObject(content, defineEvalIndex);
+		if (!extracted) {
+			searchIndex = defineEvalIndex + 10;
+			continue;
+		}
+		const id = evalIdMatchRegex.exec(extracted.objectText)?.[1];
+		if (id !== void 0) {
+			const result = {
+				filePath,
+				id
+			};
+			const title = evalTitleMatchRegex.exec(extracted.objectText)?.[1];
+			if (title !== void 0) result.title = title;
+			metas.push(result);
+		}
+		searchIndex = extracted.nextIndex;
+	}
+	return metas;
+}
+function extractDefineEvalObject(content, defineEvalIndex) {
+	const openParenIndex = content.indexOf("(", defineEvalIndex);
+	if (openParenIndex === -1) return void 0;
+	const objectStartIndex = content.indexOf("{", openParenIndex);
+	if (objectStartIndex === -1) return void 0;
+	let depth = 0;
+	let quote;
+	let inBlockComment = false;
+	let inLineComment = false;
+	let isEscaped = false;
+	for (let index = objectStartIndex; index < content.length; index++) {
+		const currentChar = content[index];
+		const nextChar = content[index + 1];
+		if (inLineComment) {
+			if (currentChar === "\n") inLineComment = false;
+			continue;
+		}
+		if (inBlockComment) {
+			if (currentChar === "*" && nextChar === "/") {
+				inBlockComment = false;
+				index++;
+			}
+			continue;
+		}
+		if (quote) {
+			if (isEscaped) {
+				isEscaped = false;
+				continue;
+			}
+			if (currentChar === "\\") {
+				isEscaped = true;
+				continue;
+			}
+			if (currentChar === quote) quote = void 0;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "/") {
+			inLineComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "*") {
+			inBlockComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "\"" || currentChar === "'" || currentChar === "`") {
+			quote = currentChar;
+			continue;
+		}
+		if (currentChar === "{") {
+			depth++;
+			continue;
+		}
+		if (currentChar === "}") {
+			depth--;
+			if (depth === 0) return {
+				nextIndex: index + 1,
+				objectText: content.slice(objectStartIndex, index + 1)
+			};
+		}
+	}
+}
+//#endregion
+//#region ../runner/src/gitState.ts
+function runGitCommand(workspaceRoot, args) {
+	const result = spawnSync("git", args, {
+		cwd: workspaceRoot,
+		encoding: "utf8",
+		stdio: [
+			"ignore",
+			"pipe",
+			"ignore"
+		]
+	});
+	return {
+		status: result.status,
+		stdout: result.stdout.trim()
+	};
+}
+/** Read the current git commit for the workspace, if available. */
+function readGitWorktreeState(workspaceRoot) {
+	const insideWorktree = runGitCommand(workspaceRoot, ["rev-parse", "--is-inside-work-tree"]);
+	if (insideWorktree.status !== 0 || insideWorktree.stdout !== "true") return { commitSha: null };
+	const commitResult = runGitCommand(workspaceRoot, ["rev-parse", "HEAD"]);
+	return { commitSha: commitResult.status === 0 ? commitResult.stdout : null };
+}
+//#endregion
+//#region ../runner/src/runChildProtocol.ts
+function isRunChildMessage(value) {
+	if (typeof value !== "object" || value === null) return false;
+	if (!("type" in value) || typeof value.type !== "string") return false;
+	if (value.type === "event") return "event" in value;
+	if (value.type === "case.finished") return "caseDetail" in value && "caseRow" in value;
+	return value.type === "done" && "evals" in value;
+}
+//#endregion
+//#region ../runner/src/runChildManager.ts
+function startRunChild(params) {
+	const child = spawn(process.execPath, [
+		...getRunChildExecArgv(),
+		resolveRunChildEntrypoint(),
+		params.contextPath
+	], {
+		cwd: params.managerContext.workspaceRoot,
+		env: process.env,
+		stdio: [
+			"ignore",
+			"inherit",
+			"inherit",
+			"ipc"
+		]
+	});
+	params.runState.childProcess = child;
+	child.on("message", (message) => {
+		if (!isRunChildMessage(message)) return;
+		handleRunChildMessage({
+			runState: params.runState,
+			message,
+			managerContext: params.managerContext
+		});
+	});
+	child.once("exit", (code, signal) => {
+		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
+		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
+		const reason = signal !== null ? `Run child exited with signal ${signal}` : `Run child exited with code ${String(code)}`;
+		markRunErrored(params.runState, reason, params.managerContext);
+	});
+}
+function getRunChildExecArgv() {
+	const execArgv = [];
+	let skipNext = false;
+	for (const arg of process.execArgv) {
+		if (skipNext) {
+			skipNext = false;
+			continue;
+		}
+		if (arg === "--eval" || arg === "-e" || arg === "--print" || arg === "-p") {
+			skipNext = true;
+			continue;
+		}
+		if (arg.startsWith("--eval=") || arg.startsWith("--print=")) continue;
+		if (arg === "--input-type" || arg.startsWith("--input-type=")) {
+			if (arg === "--input-type") skipNext = true;
+			continue;
+		}
+		execArgv.push(arg);
+	}
+	return execArgv;
+}
+function killRunChild(runState) {
+	const child = runState.childProcess;
+	runState.childProcess = void 0;
+	if (child === void 0 || child.killed) return;
+	if (!child.kill("SIGKILL")) child.kill();
+}
+function resolveRunChildEntrypoint() {
+	const currentDir = dirname(fileURLToPath(import.meta.url));
+	for (const fileName of [
+		"runChild.ts",
+		"runChild.mjs",
+		"runChild.js"
+	]) {
+		const candidate = join(currentDir, fileName);
+		if (existsSync(candidate)) return candidate;
+	}
+	throw new Error("Unable to locate the Agent Evals run child entrypoint.");
+}
+function handleRunChildMessage(params) {
+	const { runState, message, managerContext } = params;
+	if (message.type === "case.finished") {
+		if (runState.manifest.status !== "running") return;
+		upsertFinishedCase(runState, message.caseDetail, message.caseRow);
+		managerContext.emitEvent(runState, {
+			type: "case.finished",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: message.caseRow
+		});
+		return;
+	}
+	if (message.type === "done") {
+		applyChildEvalMetas(managerContext.evals, message.evals);
+		managerContext.emitDiscoveryEvent();
+		return;
+	}
+	handleRunChildEvent(runState, message.event, managerContext);
+}
+function upsertFinishedCase(runState, caseDetail, caseRow) {
+	const existingIndex = runState.cases.findIndex((row) => row.evalId === caseRow.evalId && row.caseId === caseRow.caseId && row.trial === caseRow.trial);
+	if (existingIndex === -1) runState.cases.push(caseRow);
+	else runState.cases[existingIndex] = caseRow;
+	runState.caseDetails.set(caseDetail.caseId, caseDetail);
+}
+function applyChildEvalMetas(evals, childMetas) {
+	for (const childMeta of childMetas) {
+		const evalMeta = evals.get(childMeta.id);
+		if (evalMeta === void 0) continue;
+		evalMeta.columnDefs = childMeta.columnDefs;
+		evalMeta.caseCount = childMeta.caseCount;
+		evalMeta.stats = childMeta.stats;
+		evalMeta.charts = childMeta.charts;
+		evalMeta.sourceFingerprint = childMeta.sourceFingerprint;
+	}
+}
+function handleRunChildEvent(runState, event, managerContext) {
+	if (runState.manifest.status !== "running") return;
+	if (event.type === "run.summary") {
+		const parsed = runSummarySchema.safeParse(event.payload);
+		if (parsed.success) runState.summary = parsed.data;
+		managerContext.emitEvent(runState, event);
+		return;
+	}
+	if (event.type === "run.finished") {
+		runState.childTerminalReceived = true;
+		runState.childProcess = void 0;
+		markRunTerminalFromChild(runState, event, managerContext);
+		return;
+	}
+	if (event.type === "run.error") {
+		runState.childTerminalReceived = true;
+		runState.childProcess = void 0;
+		markRunTerminalFromChild(runState, event, managerContext);
+		return;
+	}
+	managerContext.emitEvent(runState, event);
+}
+function getRunErrorMessage(payload) {
+	if (typeof payload === "object" && payload !== null && "message" in payload && typeof payload.message === "string") return payload.message;
+	return "Run child ended with an error";
+}
+async function markRunErrored(runState, message, managerContext) {
+	runState.manifest.status = "error";
+	runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+	runState.summary.status = "error";
+	runState.summary.errorMessage = message;
+	await persistRunState(runState);
+	managerContext.emitEvent(runState, {
+		type: "run.error",
+		runId: runState.manifest.id,
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		payload: { message }
+	});
+	managerContext.emitDiscoveryEvent();
+}
+async function markRunTerminalFromChild(runState, event, managerContext) {
+	const snapshot = await loadPersistedRunSnapshot(runState.runDir);
+	if (snapshot !== null) {
+		runState.manifest = snapshot.manifest;
+		runState.summary = snapshot.summary;
+		runState.cases = snapshot.cases;
+		runState.caseDetails = snapshot.caseDetails;
+	} else if (event.type === "run.finished") {
+		runState.manifest.status = "completed";
+		runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+		const parsed = runSummarySchema.safeParse(event.payload);
+		if (parsed.success) runState.summary = parsed.data;
+	} else {
+		runState.manifest.status = "error";
+		runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+		runState.summary.status = "error";
+		runState.summary.errorMessage = getRunErrorMessage(event.payload);
+	}
+	managerContext.emitEvent(runState, event);
+	managerContext.emitDiscoveryEvent();
+}
+//#endregion
+//#region ../runner/src/runner.ts
+const globMagicCharacters = new Set([
+	"*",
+	"?",
+	"[",
+	"]",
+	"{",
+	"}",
+	"(",
+	")",
+	"!",
+	"+",
+	"@"
+]);
+function hasGlobMagic(value) {
+	for (const char of value) if (globMagicCharacters.has(char)) return true;
+	return false;
+}
+function getWatchRootForIncludePattern(params) {
+	const segments = params.pattern.replaceAll("\\", "/").split("/").filter((part) => part !== "");
+	const firstGlobSegmentIndex = segments.findIndex(hasGlobMagic);
+	if (firstGlobSegmentIndex === -1) return dirname(resolve(params.workspaceRoot, params.pattern));
+	if (firstGlobSegmentIndex === 0) return params.workspaceRoot;
+	return resolve(params.workspaceRoot, segments.slice(0, firstGlobSegmentIndex).join("/"));
+}
+function getWatchRootsForIncludePatterns(params) {
+	const roots = /* @__PURE__ */ new Set();
+	for (const pattern of params.patterns) roots.add(getWatchRootForIncludePattern({
+		pattern,
+		workspaceRoot: params.workspaceRoot
+	}));
+	if (roots.size === 0) return [params.workspaceRoot];
+	return [...roots];
+}
+/** Create an in-memory eval runner bound to the current workspace config. */
+function createRunner({ watchForChanges = true } = {}) {
+	let config;
+	let workspaceRoot;
+	let localStateDir;
+	let cacheStore;
+	const evals = /* @__PURE__ */ new Map();
+	const runs = /* @__PURE__ */ new Map();
+	const lastRunStatusMap = /* @__PURE__ */ new Map();
+	const latestRunInfoMap = /* @__PURE__ */ new Map();
+	const discoveryListeners = /* @__PURE__ */ new Set();
+	let nextShortIdNum = 0;
+	let discoveryWatcher;
+	let discoveryRefreshTimer;
+	function toWorkspaceRelativePath(filePath) {
+		return relative(workspaceRoot, filePath).replaceAll("\\", "/");
+	}
+	function getSortedEvalMetas() {
+		return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+	}
+	function getSourceFingerprint(source) {
+		return createHash("sha256").update(source).digest("hex");
+	}
+	const runner = {
+		async init() {
+			config = await loadConfig();
+			workspaceRoot = config.workspaceRoot ?? process.cwd();
+			localStateDir = resolve(workspaceRoot, ".agent-evals");
+			await mkdir(localStateDir, { recursive: true });
+			await mkdir(join(localStateDir, "runs"), { recursive: true });
+			cacheStore = createFsCacheStore({
+				workspaceRoot,
+				dir: config.cache?.dir,
+				maxEntriesPerEval: config.cache?.maxEntriesPerEval
+			});
+			await loadPersistedRuns();
+			await runner.refreshDiscovery();
+			if (watchForChanges) await setupWatcher();
+		},
+		async listCache() {
+			return cacheStore.list();
+		},
+		async clearCache(filter) {
+			await cacheStore.clear(filter);
+		},
+		async recomputeStatusesForEval(evalId) {
+			const evalMeta = evals.get(evalId);
+			if (!evalMeta) return { updatedRuns: 0 };
+			const registry = getEvalRegistry();
+			await loadEvalModule(evalMeta.sourceFilePath, evalMeta.sourceFingerprint ?? void 0);
+			const entry = registry.get(evalId);
+			if (!entry) return { updatedRuns: 0 };
+			const scoreThresholds = /* @__PURE__ */ new Map();
+			entry.use((evalDef) => {
+				for (const [key, def] of Object.entries(evalDef.scores ?? {})) {
+					const threshold = normalizeScoreDef(def).passThreshold;
+					if (threshold !== void 0) scoreThresholds.set(key, threshold);
+				}
+				for (const [key, def] of Object.entries(evalDef.manualScores ?? {})) if (def.passThreshold !== void 0) scoreThresholds.set(key, def.passThreshold);
+			});
+			const updatedRuns = await recomputeEvalStatusesInRuns({
+				runs: runs.values(),
+				evalId,
+				evalExists: evals.has(evalId),
+				scoreThresholds,
+				persistCaseDetail
+			});
+			emitDiscoveryEvent();
+			return { updatedRuns };
+		},
+		async cleanRunsForEval(evalId) {
+			let deletedRuns = 0;
+			for (const [runId, run] of [...runs]) {
+				if (!runTouchesEval({
+					target: run.manifest.target,
+					caseRows: run.cases,
+					evalId,
+					evalExists: evals.has(evalId)
+				})) continue;
+				if (run.manifest.status === "running") continue;
+				runs.delete(runId);
+				await rm(run.runDir, {
+					recursive: true,
+					force: true
+				});
+				deletedRuns += 1;
+			}
+			emitDiscoveryEvent();
+			return { deletedRuns };
+		},
+		async updateManualScore({ runId, caseId, scoreKey, value }) {
+			const run = runs.get(runId);
+			if (!run) return {
+				updated: false,
+				reason: "Run not found"
+			};
+			if (run.manifest.status === "running") return {
+				updated: false,
+				reason: "Run is still running"
+			};
+			const caseRow = run.cases.find((row) => row.caseId === caseId);
+			if (!caseRow) return {
+				updated: false,
+				reason: "Case not found"
+			};
+			const evalMeta = evals.get(caseRow.evalId);
+			if (!evalMeta) return {
+				updated: false,
+				reason: "Eval not found"
+			};
+			if (evalMeta.columnDefs.find((def) => def.key === scoreKey)?.isManualScore !== true) return {
+				updated: false,
+				reason: "Manual score not found"
+			};
+			const caseDetail = run.caseDetails.get(caseId);
+			if (!caseDetail) return {
+				updated: false,
+				reason: "Case detail not found"
+			};
+			caseRow.columns[scoreKey] = value;
+			caseDetail.columns[scoreKey] = value;
+			const scoreThresholds = /* @__PURE__ */ new Map();
+			for (const def of evalMeta.columnDefs) {
+				if (def.isScore !== true || def.passThreshold === void 0) continue;
+				scoreThresholds.set(def.key, def.passThreshold);
+			}
+			const nextStatus = recomputePersistedCaseStatus(caseRow, caseDetail, scoreThresholds);
+			caseRow.status = nextStatus;
+			caseDetail.status = nextStatus;
+			const derivedSummary = deriveScopedSummaryFromCases({ caseRows: run.cases });
+			run.summary.totalCases = derivedSummary.totalCases;
+			run.summary.passedCases = derivedSummary.passedCases;
+			run.summary.failedCases = derivedSummary.failedCases;
+			run.summary.errorCases = derivedSummary.errorCases;
+			run.summary.cancelledCases = derivedSummary.cancelledCases;
+			run.summary.totalDurationMs = derivedSummary.totalDurationMs;
+			await persistCaseDetail(run.runDir, caseDetail);
+			await persistRunState(run);
+			emitDiscoveryEvent();
+			return {
+				updated: true,
+				run: {
+					manifest: run.manifest,
+					summary: run.summary,
+					cases: run.cases
+				},
+				caseDetail
+			};
+		},
+		async deleteRun(runId) {
+			const run = runs.get(runId);
+			if (!run) return { deleted: false };
+			if (run.manifest.status === "running") return { deleted: false };
+			runs.delete(runId);
+			await rm(run.runDir, {
+				recursive: true,
+				force: true
+			});
+			emitDiscoveryEvent();
+			return { deleted: true };
+		},
+		getEvals() {
+			const gitState = readGitWorktreeState(workspaceRoot);
+			const result = [];
+			for (const meta of getSortedEvalMetas()) result.push(buildEvalSummary({
+				meta,
+				config,
+				gitState,
+				latestRun: latestRunInfoMap.get(meta.id),
+				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
+			}));
+			return result;
+		},
+		getEval(id) {
+			const meta = evals.get(id);
+			if (!meta) return void 0;
+			return buildEvalSummary({
+				meta,
+				config,
+				gitState: readGitWorktreeState(workspaceRoot),
+				latestRun: latestRunInfoMap.get(meta.id),
+				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
+			});
+		},
+		async refreshDiscovery() {
+			const patterns = config.include;
+			const discovered = [];
+			for (const pattern of patterns) {
+				const files = await glob(pattern, {
+					cwd: workspaceRoot,
+					absolute: true
+				});
+				discovered.push(...files);
+			}
+			evals.clear();
+			for (const filePath of discovered) try {
+				const content = await readFile(filePath, "utf-8");
+				const discoveredMetas = parseEvalMetas(filePath, content);
+				const sourceFingerprint = getSourceFingerprint(content);
+				const registry = getEvalRegistry();
+				try {
+					await loadEvalModule(filePath, sourceFingerprint);
+				} catch {}
+				for (const meta of discoveredMetas) {
+					const discoveredEntry = registry.get(meta.id);
+					const title = meta.title;
+					let columnDefs = buildDeclaredColumnDefs(void 0, void 0, void 0);
+					let stats;
+					let charts;
+					discoveredEntry?.use((evalDef) => {
+						columnDefs = buildDeclaredColumnDefs(evalDef.columns, evalDef.scores, evalDef.manualScores);
+						stats = evalDef.stats;
+						const validated = validateCharts({
+							charts: evalDef.charts,
+							columnDefs,
+							evalId: meta.id
+						});
+						for (const warning of validated.warnings) console.warn(warning);
+						charts = validated.charts;
+					});
+					evals.set(meta.id, {
+						id: meta.id,
+						title,
+						filePath: toWorkspaceRelativePath(meta.filePath),
+						sourceFilePath: meta.filePath,
+						sourceFingerprint,
+						columnDefs,
+						caseCount: null,
+						stats,
+						charts
+					});
+				}
+			} catch {}
+			emitDiscoveryEvent();
+		},
+		async startRun(request) {
+			const runId = generateRunId();
+			const shortId = `r${String(nextShortIdNum++)}`;
+			const now = (/* @__PURE__ */ new Date()).toISOString();
+			const cacheMode = request.cache?.mode ?? "use";
+			const runDir = join(localStateDir, "runs", runId);
+			const manifest = {
+				id: runId,
+				shortId,
+				status: "running",
+				startedAt: now,
+				endedAt: null,
+				commitSha: readGitWorktreeState(workspaceRoot).commitSha,
+				evalSourceFingerprints: {},
+				target: request.target,
+				trials: request.trials,
+				trialSelection: config.trialSelection ?? "lowestScore",
+				cacheMode
+			};
+			const summary = {
+				runId,
+				status: "running",
+				totalCases: 0,
+				passedCases: 0,
+				failedCases: 0,
+				errorCases: 0,
+				cancelledCases: 0,
+				totalDurationMs: null,
+				errorMessage: null
+			};
+			const runState = {
+				runDir,
+				manifest,
+				summary,
+				cases: [],
+				caseDetails: /* @__PURE__ */ new Map(),
+				listeners: /* @__PURE__ */ new Set(),
+				childProcess: void 0,
+				childTerminalReceived: false
+			};
+			runs.set(runId, runState);
+			setLatestRunInfoMap({
+				latestRunInfoMap,
+				evalIds: getTargetEvalIds({
+					request,
+					sortedEvalIds: getSortedEvalMetas().map((meta) => meta.id),
+					knownEvalIds: new Set(evals.keys())
+				}),
+				info: {
+					status: "running",
+					startedAt: now,
+					commitSha: manifest.commitSha ?? null,
+					evalSourceFingerprint: null
+				}
+			});
+			await mkdir(runDir, { recursive: true });
+			await mkdir(join(runDir, "traces"), { recursive: true });
+			await mkdir(join(runDir, "artifacts"), { recursive: true });
+			await mkdir(join(runDir, "case-details"), { recursive: true });
+			await writeFile(join(runDir, "run.json"), JSON.stringify(manifest, null, 2));
+			const childContext = {
+				request,
+				workspaceRoot,
+				runDir,
+				manifest,
+				summary,
+				evals: getSortedEvalMetas()
+			};
+			await writeFile(join(runDir, "run-child-context.json"), JSON.stringify(childContext, null, 2));
+			startRunChild({
+				runState,
+				contextPath: join(runDir, "run-child-context.json"),
+				managerContext: {
+					workspaceRoot,
+					evals,
+					emitEvent,
+					emitDiscoveryEvent
+				}
+			});
+			return {
+				manifest,
+				summary,
+				cases: []
+			};
+		},
+		getRuns() {
+			return [...runs.values()].map((r) => r.manifest);
+		},
+		getRun(id) {
+			const run = runs.get(id);
+			if (!run) return void 0;
+			return {
+				manifest: run.manifest,
+				summary: run.summary,
+				cases: run.cases
+			};
+		},
+		async cancelRun(id) {
+			const run = runs.get(id);
+			if (!run) return;
+			if (run.manifest.status !== "running") return;
+			const endedAt = /* @__PURE__ */ new Date();
+			run.manifest.status = "cancelled";
+			run.manifest.endedAt = endedAt.toISOString();
+			run.summary.status = "cancelled";
+			const derivedSummary = deriveScopedSummaryFromCases({
+				caseRows: run.cases,
+				lifecycleStatus: "cancelled"
+			});
+			run.summary.totalCases = derivedSummary.totalCases;
+			run.summary.passedCases = derivedSummary.passedCases;
+			run.summary.failedCases = derivedSummary.failedCases;
+			run.summary.errorCases = derivedSummary.errorCases;
+			run.summary.cancelledCases = derivedSummary.cancelledCases;
+			run.summary.totalDurationMs = endedAt.getTime() - new Date(run.manifest.startedAt).getTime();
+			killRunChild(run);
+			await persistRunState(run);
+			emitEvent(run, {
+				type: "run.cancelled",
+				runId: id,
+				timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+				payload: run.summary
+			});
+			emitDiscoveryEvent();
+		},
+		getCaseDetail(runId, caseId) {
+			const run = runs.get(runId);
+			if (!run) return void 0;
+			return run.caseDetails.get(caseId);
+		},
+		subscribe(runId, listener) {
+			const run = runs.get(runId);
+			if (!run) return () => {};
+			run.listeners.add(listener);
+			return () => {
+				run.listeners.delete(listener);
+			};
+		},
+		subscribeDiscovery(listener) {
+			discoveryListeners.add(listener);
+			return () => {
+				discoveryListeners.delete(listener);
+			};
+		},
+		async close() {
+			if (discoveryRefreshTimer !== void 0) {
+				clearTimeout(discoveryRefreshTimer);
+				discoveryRefreshTimer = void 0;
+			}
+			const watcher = discoveryWatcher;
+			if (watcher === void 0) return;
+			discoveryWatcher = void 0;
+			await watcher.close();
+		},
+		getWorkspaceRoot() {
+			return workspaceRoot;
+		},
+		getArtifactPath(artifactId_) {
+			return resolveArtifactPath(join(localStateDir, "runs"), artifactId_);
+		}
+	};
+	async function setupWatcher() {
+		const watcher = watch(getWatchRootsForIncludePatterns({
+			patterns: config.include,
+			workspaceRoot
+		}), {
+			ignoreInitial: true,
+			persistent: true
+		});
+		discoveryWatcher = watcher;
+		const scheduleRefresh = () => {
+			if (discoveryRefreshTimer !== void 0) clearTimeout(discoveryRefreshTimer);
+			discoveryRefreshTimer = setTimeout(() => {
+				discoveryRefreshTimer = void 0;
+				runner.refreshDiscovery();
+			}, 50);
+		};
+		watcher.on("change", scheduleRefresh);
+		watcher.on("add", scheduleRefresh);
+		watcher.on("unlink", scheduleRefresh);
+		watcher.on("addDir", scheduleRefresh);
+		watcher.on("unlinkDir", scheduleRefresh);
+		await new Promise((ready) => {
+			watcher.once("ready", ready);
+		});
+	}
+	function emitDiscoveryEvent() {
+		const lastRunStatuses = getLastRunStatuses({
+			runs: runs.values(),
+			knownEvals: evals.values()
+		});
+		const latestRunInfos = getLatestRunInfos({
+			runs: runs.values(),
+			knownEvals: evals.values()
+		});
+		lastRunStatusMap.clear();
+		for (const [evalId, status] of lastRunStatuses) lastRunStatusMap.set(evalId, status);
+		latestRunInfoMap.clear();
+		for (const [evalId, info] of latestRunInfos) latestRunInfoMap.set(evalId, info);
+		const event = {
+			type: "discovery.updated",
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: runner.getEvals()
+		};
+		for (const listener of discoveryListeners) listener(event);
+	}
+	function emitEvent(runState, event) {
+		for (const listener of runState.listeners) try {
+			listener(event);
+		} catch {}
+	}
+	async function loadPersistedRuns() {
+		runs.clear();
+		const persistedRuns = await loadPersistedRunSnapshots(localStateDir);
+		nextShortIdNum = nextShortIdFromSnapshots(persistedRuns);
+		for (const persistedRun of persistedRuns) runs.set(persistedRun.manifest.id, {
+			...persistedRun,
+			listeners: /* @__PURE__ */ new Set(),
+			childProcess: void 0,
+			childTerminalReceived: false
+		});
+	}
+	return runner;
+}
+//#endregion
+//#region src/cli.ts
+function parseArgs(argv) {
+	const normalizedArgv = argv.filter((arg) => arg !== "--no-env");
+	const args = {
+		command: "help",
+		subcommand: void 0,
+		showHelp: false,
+		helpTopic: "global",
+		unknownHelpTarget: void 0,
+		evalIds: [],
+		caseIds: [],
+		trials: 1,
+		json: false,
+		port: 4100,
+		cacheMode: "use",
+		clearCache: false,
+		all: false,
+		loadEnv: normalizedArgv.length === argv.length
+	};
+	const command = normalizedArgv[0];
+	if (command === "--help" || command === "-h") {
+		args.showHelp = true;
+		return args;
+	}
+	if (isCliCommand(command)) {
+		args.command = command;
+		args.helpTopic = command === "help" ? "global" : command;
+	} else if (command !== void 0 && !command.startsWith("-")) args.unknownHelpTarget = command;
+	let cursor = 1;
+	if (args.command === "cache") {
+		const sub = normalizedArgv[cursor];
+		if (sub === "list" || sub === "clear") {
+			args.subcommand = sub;
+			args.helpTopic = `cache ${sub}`;
+			cursor++;
+		} else if (sub !== void 0 && !sub.startsWith("-")) args.unknownHelpTarget = `cache ${sub}`;
+	}
+	for (let i = cursor; i < normalizedArgv.length; i++) {
+		const arg = normalizedArgv[i];
+		const next = normalizedArgv[i + 1];
+		if (arg === "--help" || arg === "-h") args.showHelp = true;
+		else if (arg === "--eval" && next) {
+			args.evalIds.push(...next.split(","));
+			i++;
+		} else if (arg === "--case" && next) {
+			args.caseIds.push(...next.split(","));
+			i++;
+		} else if (arg === "--trials" && next) {
+			args.trials = Number(next);
+			i++;
+		} else if (arg === "--json") args.json = true;
+		else if (arg === "--port" && next) {
+			args.port = Number(next);
+			i++;
+		} else if (arg === "--cache" && next) {
+			if (next === "use" || next === "bypass" || next === "refresh") args.cacheMode = next;
+			i++;
+		} else if (arg === "--no-cache") args.cacheMode = "bypass";
+		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
+		else if (arg === "--clear-cache") args.clearCache = true;
+		else if (arg === "--all") args.all = true;
+	}
+	return args;
+}
+/**
+* Run the Agent Evals CLI against the current workspace.
+*
+* @param argv Raw command-line arguments excluding the executable name.
+*/
+async function runCli(argv) {
+	const args = parseArgs(argv);
+	if (args.loadEnv && !loadWorkspaceEnv()) {
+		process.exit(1);
+		return;
+	}
+	if (args.showHelp) {
+		if (args.unknownHelpTarget !== void 0) {
+			console.error(`No help found for "${args.unknownHelpTarget}".`);
+			process.exit(1);
+			return;
+		}
+		printHelp(args.helpTopic);
+		return;
+	}
+	switch (args.command) {
+		case "app":
+			await commandApp(args);
+			break;
+		case "list":
+			await commandList(args);
+			break;
+		case "run":
+			await commandRun(args);
+			break;
+		case "cache":
+			await commandCache(args);
+			break;
+		default:
+			printHelp(args.helpTopic);
+			break;
+	}
+}
+function isCliCommand(command) {
+	return command === "app" || command === "list" || command === "run" || command === "cache" || command === "help";
+}
+function loadWorkspaceEnv() {
+	const envPath = resolve(process.cwd(), ".env");
+	if (!existsSync(envPath)) return true;
+	const loadResult = resultify(() => {
+		process.loadEnvFile(envPath);
+	});
+	if (loadResult.error) {
+		console.error(`Failed to load .env at ${envPath}: ${loadResult.error.message}`);
+		return false;
+	}
+	return true;
+}
+const currentDir = dirname(fileURLToPath(import.meta.url));
+const repoRoot = resolve(currentDir, "../../..");
+const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
+function hasRepoWebWorkspace() {
+	return existsSync(resolve(repoRoot, "apps/web/package.json"));
+}
+async function ensureWebUiIsBuilt() {
+	if (!hasRepoWebWorkspace()) return;
+	console.info("Preparing web UI...");
+	await new Promise((resolvePromise, rejectPromise) => {
+		const child = spawn(pnpmCommand, [
+			"--filter",
+			"@agent-evals/web",
+			"build"
+		], {
+			cwd: repoRoot,
+			stdio: "inherit"
+		});
+		child.once("error", (error) => {
+			rejectPromise(error);
+		});
+		child.once("exit", (code, signal) => {
+			if (signal) {
+				rejectPromise(/* @__PURE__ */ new Error(`Web UI build stopped with signal ${signal}.`));
+				return;
+			}
+			if (code !== 0) {
+				rejectPromise(/* @__PURE__ */ new Error(`Web UI build failed with exit code ${String(code)}.`));
+				return;
+			}
+			resolvePromise();
+		});
+	});
+}
+function isHonoAppModule(mod) {
+	if (typeof mod !== "object" || mod === null || !("app" in mod)) return false;
+	const { app } = mod;
+	return typeof app === "object" && app !== null && "fetch" in app && typeof app.fetch === "function";
+}
+function isServerRunnerModule(mod) {
+	if (typeof mod !== "object" || mod === null || !("initRunner" in mod)) return false;
+	return typeof mod.initRunner === "function";
+}
+async function commandApp(args) {
+	await ensureWebUiIsBuilt();
+	const { serve } = await import("@hono/node-server");
+	const bundledWebDist = resolve(currentDir, "apps/web/dist");
+	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
+	const appModule = await import("./app-TjV5nDMM.mjs");
+	const runnerModule = await import("./runner-CBDZos0Z.mjs");
+	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
+	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
+	await runnerModule.initRunner();
+	console.info(`Agent Evals app: http://localhost:${String(args.port)}`);
+	serve({
+		fetch: appModule.app.fetch,
+		port: args.port
+	});
+}
+async function commandList(args_) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	const evals = runner.getEvals();
+	if (evals.length === 0) {
+		console.info("No eval files found.");
+		return;
+	}
+	console.info("Discovered evals:\n");
+	for (const ev of evals) {
+		const displayStatus = getEvalDisplayStatus({
+			freshnessStatus: ev.freshnessStatus,
+			stale: ev.stale,
+			outdated: ev.outdated,
+			lastRunStatus: ev.lastRunStatus
+		});
+		const title = getEvalTitle(ev);
+		console.info(`  ${title}`);
+		console.info(`    id: ${ev.id}`);
+		console.info(`    file: ${ev.filePath}`);
+		if (displayStatus !== "pending") console.info(`    status: ${displayStatus}`);
+		if (ev.caseCount !== null) console.info(`    cases: ${String(ev.caseCount)}`);
+		console.info("");
+	}
+}
+async function commandRun(args) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	if (args.clearCache) {
+		await runner.clearCache();
+		if (!args.json) {
+			console.info("Cleared cache before run.");
+			console.info("");
+		}
+	}
+	const target = args.caseIds.length > 0 ? {
+		mode: "caseIds",
+		caseIds: args.caseIds,
+		evalIds: args.evalIds.length > 0 ? args.evalIds : void 0
+	} : args.evalIds.length > 0 ? {
+		mode: "evalIds",
+		evalIds: args.evalIds
+	} : { mode: "all" };
+	const run = await runner.startRun({
+		target,
+		trials: args.trials,
+		cache: { mode: args.cacheMode }
+	});
+	if (!args.json) {
+		console.info(`Run started: ${run.manifest.id}`);
+		console.info(`Trials: ${String(args.trials)}`);
+		if (args.cacheMode !== "use") console.info(`Cache mode: ${args.cacheMode}`);
+		console.info("");
+	}
+	await waitForRunCompletion(runner, run.manifest.id);
+	const finalRun = runner.getRun(run.manifest.id);
+	if (!finalRun) {
+		process.exit(1);
+		return;
+	}
+	const { summary } = finalRun;
+	if (args.json) console.info(JSON.stringify(summary, null, 2));
+	else {
+		console.info("--- Run Summary ---");
+		console.info(`Status: ${summary.status}`);
+		console.info(`Total: ${String(summary.totalCases)}`);
+		console.info(`Passed: ${String(summary.passedCases)}`);
+		console.info(`Failed: ${String(summary.failedCases)}`);
+		console.info(`Errors: ${String(summary.errorCases)}`);
+		if (summary.totalCases > 0) console.info(`Pass Rate: ${String(summary.passedCases)}/${String(summary.totalCases)}`);
+		if (summary.totalDurationMs !== null) console.info(`Duration: ${(summary.totalDurationMs / 1e3).toFixed(1)}s`);
+	}
+	if (summary.failedCases > 0 || summary.errorCases > 0) process.exit(1);
+}
+async function commandCache(args) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	if (args.subcommand === "list" || args.subcommand === void 0) {
+		const entries = await runner.listCache();
+		if (args.json) {
+			console.info(JSON.stringify(entries, null, 2));
+			return;
+		}
+		if (entries.length === 0) {
+			console.info("No cache entries.");
+			return;
+		}
+		console.info(`Cache entries (${String(entries.length)}):\n`);
+		for (const entry of entries) {
+			console.info(`  ${entry.namespace}`);
+			console.info(`    key: ${entry.key}`);
+			const operationLabel = entry.operationType === "span" ? `${entry.operationName} (span ${entry.spanKind ?? "unknown"})` : `${entry.operationName} (value)`;
+			console.info(`    operation: ${operationLabel}`);
+			console.info(`    stored: ${entry.storedAt}`);
+			console.info(`    size: ${String(entry.sizeBytes)} bytes`);
+			console.info("");
+		}
+		return;
+	}
+	if (args.subcommand === "clear") {
+		if (args.evalIds.length > 0) {
+			for (const evalId of args.evalIds) {
+				const entries = await runner.listCache();
+				const prefix = `${evalId}__`;
+				const matching = entries.filter((entry) => entry.namespace.startsWith(prefix));
+				for (const entry of matching) await runner.clearCache({
+					namespace: entry.namespace,
+					key: entry.key
+				});
+			}
+			console.info(`Cleared cache entries for: ${args.evalIds.join(", ")}`);
+			return;
+		}
+		if (args.all) {
+			await runner.clearCache();
+			console.info("Cleared all cache entries.");
+			return;
+		}
+		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
+		process.exit(1);
+		return;
+	}
+	printHelp(args.helpTopic);
+}
+async function waitForRunCompletion(runner, runId) {
+	return new Promise((resolvePromise) => {
+		const check = () => {
+			const run = runner.getRun(runId);
+			if (!run || run.manifest.status === "completed" || run.manifest.status === "cancelled" || run.manifest.status === "error") {
+				resolvePromise();
+				return;
+			}
+			setTimeout(check, 200);
+		};
+		check();
+	});
+}
+function printHelp(topic = "global") {
+	if (topic === "app") {
+		console.info(`
+agent-evals app - Start server with UI
+
+Usage:
+  agent-evals app [flags]
+
+Flags:
+  --port <n>                 Server port (default: 4100)
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "list") {
+		console.info(`
+agent-evals list - List discovered evals
+
+Usage:
+  agent-evals list [flags]
+
+Flags:
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "run") {
+		console.info(`
+agent-evals run - Run evals
+
+Usage:
+  agent-evals run [flags]
+
+Flags:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --case <id>                Run specific case(s) (comma-separated)
+  --trials <n>               Number of trials per case
+  --inspect[=host:port]      Run with the Node.js inspector enabled
+  --inspect-brk[=host:port]  Enable inspector and pause before startup
+  --json                     Output run summary as JSON
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
+		console.info(`
+agent-evals cache - Manage cached operation entries
+
+Usage:
+  agent-evals cache list [flags]
+  agent-evals cache clear --eval <id>
+  agent-evals cache clear --all
+
+Flags:
+  --eval <id>                Clear entries for specific eval(s) (comma-separated)
+  --all                      Confirm clearing every cached entry
+  --json                     Output cache listing as JSON
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show this help
+  `);
+		return;
+	}
+	console.info(`
+agent-evals - LLM/Agent eval runner
+
+Commands:
+  app                        Start server with UI
+  list                       List discovered evals
+  run                        Run evals
+  cache list                 List cached operation entries
+  cache clear --eval <id>    Clear cache entries for one eval
+  cache clear --all          Clear every cached entry
+  help                       Show this help
+
+Options:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --case <id>                Run specific case(s) (comma-separated)
+  --trials <n>               Number of trials per case
+  --inspect[=host:port]      Run with the Node.js inspector enabled
+  --inspect-brk[=host:port]  Enable inspector and pause before startup
+  --json                     Output results as JSON
+  --port <n>                 Server port (default: 4100)
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  --no-env                   Disable automatic .env loading
+  --help, -h                 Show help
+  `);
+}
+//#endregion
+export { createRunner as n, runCli as t };