@ls-stack/agent-eval 0.2.0

package/dist/cli-CwEFLP0w.mjs

@@ -0,0 +1,3422 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { dirname, extname, join, relative, resolve } from "node:path";
+import { AsyncLocalStorage } from "node:async_hooks";
+import { getCompositeKey } from "@ls-stack/utils/getCompositeKey";
+import { z } from "zod/v4";
+import { watch } from "chokidar";
+import { glob } from "glob";
+import { existsSync } from "node:fs";
+import { resultify } from "t-result";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { spawn, spawnSync } from "node:child_process";
+//#region ../sdk/src/defineEval.ts
+const evalRegistry = /* @__PURE__ */ new Map();
+/** Return the in-memory registry of evals defined in the current process. */
+function getEvalRegistry() {
+	return evalRegistry;
+}
+/**
+* Register an eval definition with the SDK so the runner can discover it
+* after importing the eval module.
+*/
+function defineEval(definition) {
+	evalRegistry.set(definition.id, {
+		id: definition.id,
+		title: definition.title,
+		use: (fn) => fn(definition)
+	});
+}
+//#endregion
+//#region ../sdk/src/repoFile.ts
+/**
+* Create a file reference that can be emitted via `setEvalOutput(...)` and rendered
+* by a column configured with `format: 'image' | 'audio' | 'video' | 'file'`.
+*
+* @param path Relative or absolute path to the repository file.
+* @param mimeType Optional MIME type hint for UI rendering.
+* @returns A repo-backed file reference suitable for file/media columns.
+*/
+function repoFile(path, mimeType) {
+	return {
+		source: "repo",
+		path,
+		mimeType
+	};
+}
+//#endregion
+//#region ../sdk/src/runtime.ts
+const scopeStorage = new AsyncLocalStorage();
+let activeEvalScopeCount = 0;
+/** Error thrown when an eval assertion fails during case execution. */
+var EvalAssertionError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "EvalAssertionError";
+	}
+};
+/** Return the current eval scope for the active async context, if any. */
+function getCurrentScope() {
+	if (activeEvalScopeCount === 0) return void 0;
+	return scopeStorage.getStore();
+}
+/**
+* Return whether the current async execution is inside an active eval case.
+*
+* This is useful for shared workflow code that wants to branch on eval-only
+* behavior without importing or inspecting the full eval scope.
+*/
+function isInEvalScope() {
+	return getCurrentScope() !== void 0;
+}
+/**
+* Attach cache context (adapter, mode, eval id, fingerprint) to a scope.
+*
+* Runner-internal helper called immediately before the user's `execute`
+* function runs inside `runInEvalScope`.
+*/
+function setScopeCacheContext(scope, context) {
+	scope.cacheContext = context;
+}
+/**
+* Execute a callback inside a fresh eval case scope and capture its outputs,
+* trace data, and terminal error state.
+*/
+async function runInEvalScope(caseId, fn, options = {}) {
+	const scope = {
+		caseId,
+		outputs: {},
+		assertionFailures: [],
+		spans: [],
+		checkpoints: /* @__PURE__ */ new Map(),
+		spanStack: [],
+		activeSpanStack: [],
+		recordingStack: [],
+		replayingDepth: 0,
+		cacheContext: options.cacheContext
+	};
+	activeEvalScopeCount++;
+	try {
+		return await scopeStorage.run(scope, async () => {
+			try {
+				return {
+					result: await fn(),
+					scope,
+					error: void 0
+				};
+			} catch (error) {
+				return {
+					result: void 0,
+					scope,
+					error: error instanceof Error ? error : new Error(String(error))
+				};
+			}
+		});
+	} finally {
+		activeEvalScopeCount--;
+	}
+}
+function recordOpIfActive(scope, op) {
+	if (scope.replayingDepth > 0) return;
+	const top = scope.recordingStack.at(-1);
+	if (top) top.ops.push(op);
+}
+function toAssertionFailure$1(message, error = void 0) {
+	return error?.stack ? {
+		message,
+		stack: error.stack
+	} : { message };
+}
+/**
+* Record or replace an output value for the current case scope.
+*
+* Supported values include scalars, JSON-safe objects/arrays, explicit file
+* refs, and native `Blob`/`File` instances for media or file columns.
+*/
+function setEvalOutput(key, value) {
+	const scope = getCurrentScope();
+	if (!scope) return;
+	scope.outputs[key] = value;
+	recordOpIfActive(scope, {
+		kind: "setOutput",
+		key,
+		value
+	});
+}
+/**
+* Add a numeric delta to an output value in the current case scope.
+*
+* If the existing value is non-numeric, the operation is recorded as an
+* assertion failure instead of mutating the output.
+*/
+function incrementEvalOutput(key, delta) {
+	const scope = getCurrentScope();
+	if (!scope) return;
+	const existing = scope.outputs[key];
+	if (existing === void 0) {
+		scope.outputs[key] = delta;
+		recordOpIfActive(scope, {
+			kind: "incrementOutput",
+			key,
+			delta
+		});
+		return;
+	}
+	if (typeof existing !== "number") {
+		scope.assertionFailures.push(toAssertionFailure$1(`incrementEvalOutput("${key}"): existing value is ${typeof existing}, expected number`));
+		return;
+	}
+	scope.outputs[key] = existing + delta;
+	recordOpIfActive(scope, {
+		kind: "incrementOutput",
+		key,
+		delta
+	});
+}
+/**
+* Assert a condition for the current eval case and throw on failure.
+*
+* Calls made outside `runInEvalScope(...)` are ignored so shared workflow code
+* can safely reuse `evalAssert(...)` when it also runs outside an eval.
+*/
+function evalAssert(condition, message) {
+	if (condition) return;
+	const scope = getCurrentScope();
+	if (!scope) return;
+	const error = new EvalAssertionError(message);
+	scope.assertionFailures.push(toAssertionFailure$1(message, error));
+	throw error;
+}
+//#endregion
+//#region ../sdk/src/tracer.ts
+let spanIdCounter = 0;
+function generateSpanId() {
+	spanIdCounter++;
+	return `span_${String(Date.now())}_${String(spanIdCounter)}`;
+}
+function updateCurrentSpan(update) {
+	const currentSpan = getCurrentScope()?.activeSpanStack.at(-1);
+	if (!currentSpan) return;
+	update(currentSpan);
+}
+function noopActiveSpan() {
+	return {
+		setName() {},
+		setAttribute() {},
+		setAttributes() {}
+	};
+}
+function mergeSpanAttributes(span, attributes) {
+	span.attributes = {
+		...span.attributes,
+		...attributes
+	};
+}
+function createSpanHandle(span) {
+	return {
+		setName(value) {
+			span.name = value;
+		},
+		setAttribute(key, value) {
+			mergeSpanAttributes(span, { [key]: value });
+		},
+		setAttributes(value) {
+			mergeSpanAttributes(span, value);
+		}
+	};
+}
+/**
+* Ambient handle for the active span in the current async context.
+*
+* Calls are no-ops when executed outside of `evalTracer.span(...)`.
+*/
+const evalSpan = {
+	setName(value) {
+		updateCurrentSpan((currentSpan) => {
+			currentSpan.name = value;
+		});
+	},
+	setAttribute(key, value) {
+		updateCurrentSpan((currentSpan) => {
+			mergeSpanAttributes(currentSpan, { [key]: value });
+		});
+	},
+	setAttributes(value) {
+		updateCurrentSpan((currentSpan) => {
+			mergeSpanAttributes(currentSpan, value);
+		});
+	}
+};
+async function traceSpan(info, fn) {
+	const scope = getCurrentScope();
+	if (!scope) return await fn(noopActiveSpan());
+	const id = generateSpanId();
+	const spanRecord = {
+		id,
+		parentId: scope.activeSpanStack.at(-1)?.id ?? null,
+		caseId: scope.caseId,
+		kind: info.kind,
+		name: info.name,
+		startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		endedAt: null,
+		status: "running",
+		attributes: info.attributes
+	};
+	scope.spans.push(spanRecord);
+	scope.spanStack.push(id);
+	scope.activeSpanStack.push(spanRecord);
+	const activeSpan = createSpanHandle(spanRecord);
+	try {
+		const cacheOpts = info.cache;
+		const cacheCtx = scope.cacheContext;
+		if (cacheOpts !== void 0 && cacheCtx !== void 0 && scope.replayingDepth === 0) {
+			const ctx = cacheCtx;
+			const namespace = cacheOpts.namespace ?? `${ctx.evalId}__${info.name}`;
+			const keyHash = hashCacheKey({
+				namespace,
+				codeFingerprint: ctx.codeFingerprint,
+				key: cacheOpts.key
+			});
+			mergeSpanAttributes(spanRecord, {
+				"cache.key": keyHash,
+				"cache.namespace": namespace
+			});
+			if (ctx.mode === "use") {
+				const hit = await ctx.adapter.lookup(namespace, keyHash);
+				if (hit) {
+					const storedAt = hit.storedAt;
+					mergeSpanAttributes(spanRecord, {
+						"cache.status": "hit",
+						"cache.storedAt": storedAt,
+						"cache.age": Date.now() - new Date(storedAt).getTime()
+					});
+					replayRecording(scope, spanRecord, hit.recording);
+					spanRecord.status = "ok";
+					spanRecord.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+					return hit.recording.returnValue;
+				}
+				mergeSpanAttributes(spanRecord, { "cache.status": "miss" });
+			} else if (ctx.mode === "refresh") mergeSpanAttributes(spanRecord, { "cache.status": "refresh" });
+			else mergeSpanAttributes(spanRecord, { "cache.status": "bypass" });
+			const frame = {
+				baseSpanIndex: scope.spans.length,
+				cachedSpanId: id,
+				ops: []
+			};
+			scope.recordingStack.push(frame);
+			let bodyResult;
+			try {
+				bodyResult = await fn(activeSpan);
+			} finally {
+				scope.recordingStack.pop();
+			}
+			appendSubSpanOps(scope, frame);
+			if (ctx.mode !== "bypass") {
+				const recording = {
+					returnValue: toJsonSafe(bodyResult),
+					finalAttributes: stripCacheAttributes(spanRecord.attributes),
+					ops: frame.ops
+				};
+				const entry = {
+					version: 1,
+					key: keyHash,
+					namespace,
+					spanName: info.name,
+					spanKind: info.kind,
+					storedAt: (/* @__PURE__ */ new Date()).toISOString(),
+					codeFingerprint: ctx.codeFingerprint,
+					recording
+				};
+				await ctx.adapter.write(entry);
+			}
+			spanRecord.status = "ok";
+			spanRecord.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+			return bodyResult;
+		}
+		const result = await fn(activeSpan);
+		spanRecord.status = "ok";
+		spanRecord.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+		return result;
+	} catch (error) {
+		spanRecord.status = "error";
+		spanRecord.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+		if (error instanceof Error) spanRecord.error = {
+			name: error.name,
+			message: error.message,
+			stack: error.stack
+		};
+		else spanRecord.error = { message: String(error) };
+		throw error;
+	} finally {
+		scope.spanStack.pop();
+		scope.activeSpanStack.pop();
+	}
+}
+/**
+* Trace builder used to create hierarchical spans and checkpoints during eval
+* execution.
+*/
+const evalTracer = {
+	/** Run a callback inside a new trace span and record its lifecycle. */
+	span: traceSpan,
+	/** Record a named point-in-time value alongside the trace. */
+	checkpoint(name, data) {
+		const scope = getCurrentScope();
+		if (!scope) return;
+		scope.checkpoints.set(name, data);
+		const id = generateSpanId();
+		const parentId = scope.spanStack.at(-1) ?? null;
+		scope.spans.push({
+			id,
+			parentId,
+			caseId: scope.caseId,
+			kind: "checkpoint",
+			name,
+			startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			endedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			status: "ok",
+			attributes: { value: data }
+		});
+		if (scope.replayingDepth === 0) {
+			const top = scope.recordingStack.at(-1);
+			if (top) top.ops.push({
+				kind: "checkpoint",
+				name,
+				data
+			});
+		}
+	}
+};
+/** Build a queryable trace tree helper from a flat span list and checkpoints. */
+function buildTraceTree(spans, checkpoints) {
+	return {
+		spans,
+		rootSpans: spans.filter((s) => s.parentId === null),
+		findSpan(name) {
+			return spans.find((s) => s.name === name);
+		},
+		findSpansByKind(kind) {
+			return spans.filter((s) => s.kind === kind);
+		},
+		flattenDfs() {
+			const result = [];
+			function visit(parentId) {
+				for (const childSpan of spans) if (childSpan.parentId === parentId) {
+					result.push(childSpan);
+					visit(childSpan.id);
+				}
+			}
+			visit(null);
+			return result;
+		},
+		checkpoints
+	};
+}
+/** Hash the components of a cache key into a deterministic hex digest. */
+function hashCacheKey(input) {
+	return createHash("sha256").update(getCompositeKey(input)).digest("hex");
+}
+function toJsonSafe(value) {
+	if (value === void 0) return void 0;
+	const text = JSON.stringify(value);
+	return JSON.parse(text);
+}
+function stripCacheAttributes(attributes) {
+	if (!attributes) return {};
+	const result = {};
+	for (const [key, value] of Object.entries(attributes)) if (!key.startsWith("cache.")) result[key] = value;
+	return result;
+}
+function serializeSubSpanTree(scope, spanId) {
+	const original = scope.spans.find((s) => s.id === spanId);
+	if (!original) return {
+		kind: "custom",
+		name: "unknown",
+		attributes: void 0,
+		status: "ok",
+		error: void 0,
+		children: []
+	};
+	const children = scope.spans.filter((s) => s.parentId === spanId).map((child) => serializeSubSpanTree(scope, child.id));
+	return {
+		kind: original.kind,
+		name: original.name,
+		attributes: original.attributes,
+		status: original.status,
+		error: original.error,
+		children
+	};
+}
+function appendSubSpanOps(scope, frame) {
+	for (let i = frame.baseSpanIndex; i < scope.spans.length; i++) {
+		const candidate = scope.spans[i];
+		if (candidate?.parentId === frame.cachedSpanId) frame.ops.push({
+			kind: "subSpan",
+			span: serializeSubSpanTree(scope, candidate.id)
+		});
+	}
+}
+function replayRecording(scope, parentSpan, recording) {
+	scope.replayingDepth++;
+	try {
+		for (const op of recording.ops) applyRecordingOp(scope, parentSpan, op);
+		if (Object.keys(recording.finalAttributes).length > 0) mergeSpanAttributes(parentSpan, recording.finalAttributes);
+	} finally {
+		scope.replayingDepth--;
+	}
+}
+function applyRecordingOp(scope, parentSpan, op) {
+	if (op.kind === "setOutput") {
+		scope.outputs[op.key] = op.value;
+		return;
+	}
+	if (op.kind === "incrementOutput") {
+		const existing = scope.outputs[op.key];
+		if (existing === void 0) scope.outputs[op.key] = op.delta;
+		else if (typeof existing === "number") scope.outputs[op.key] = existing + op.delta;
+		else scope.assertionFailures.push({ message: `replay incrementEvalOutput("${op.key}"): existing value is ${typeof existing}, expected number` });
+		return;
+	}
+	if (op.kind === "checkpoint") {
+		scope.checkpoints.set(op.name, op.data);
+		return;
+	}
+	replaySerializedSpan(scope, parentSpan.id, op.span);
+}
+function replaySerializedSpan(scope, parentId, serialized) {
+	const id = generateSpanId();
+	const now = (/* @__PURE__ */ new Date()).toISOString();
+	const replayed = {
+		id,
+		parentId,
+		caseId: scope.caseId,
+		kind: serialized.kind,
+		name: serialized.name,
+		startedAt: now,
+		endedAt: now,
+		status: serialized.status,
+		attributes: serialized.attributes,
+		error: serialized.error
+	};
+	scope.spans.push(replayed);
+	for (const child of serialized.children) replaySerializedSpan(scope, id, child);
+}
+//#endregion
+//#region ../shared/src/schemas/display.ts
+const scalarCellSchema = z.union([
+	z.string(),
+	z.number(),
+	z.boolean(),
+	z.null()
+]);
+const jsonCellSchema = z.lazy(() => z.union([
+	scalarCellSchema,
+	z.array(jsonCellSchema),
+	z.record(z.string(), jsonCellSchema)
+]));
+const repoFileRefSchema = z.object({
+	source: z.literal("repo"),
+	path: z.string(),
+	mimeType: z.string().optional()
+});
+const runArtifactRefSchema = z.object({
+	source: z.literal("run"),
+	artifactId: z.string(),
+	mimeType: z.string(),
+	fileName: z.string().optional()
+});
+const fileRefSchema = z.union([repoFileRefSchema, runArtifactRefSchema]);
+/** Schema for numeric presentation options used by number-formatted values. */
+const numberDisplayOptionsSchema = z.object({
+	notation: z.enum(["standard", "compact"]).optional(),
+	compactDisplay: z.enum(["short", "long"]).optional(),
+	prefix: z.string().optional(),
+	suffix: z.string().optional(),
+	decimalPlaces: z.number().int().min(0).optional()
+});
+/** Schema for the supported column rendering kinds in list views. */
+const columnKindSchema = z.enum([
+	"string",
+	"number",
+	"boolean"
+]);
+/** Schema for the built-in column formatting presets. */
+const columnFormatSchema = z.enum([
+	"boolean",
+	"markdown",
+	"json",
+	"image",
+	"audio",
+	"video",
+	"file",
+	"duration",
+	"percent",
+	"number",
+	"passFail",
+	"stars"
+]);
+/** Schema describing a rendered column in the eval results table. */
+const columnDefSchema = z.object({
+	key: z.string(),
+	label: z.string(),
+	kind: columnKindSchema,
+	format: columnFormatSchema.optional(),
+	numberFormat: numberDisplayOptionsSchema.optional(),
+	isScore: z.boolean().optional(),
+	isManualScore: z.boolean().optional(),
+	passThreshold: z.number().optional(),
+	maxStars: z.number().int().min(2).optional(),
+	hideInTable: z.boolean().optional(),
+	sortable: z.boolean().optional(),
+	align: z.enum([
+		"left",
+		"center",
+		"right"
+	]).optional()
+});
+/** Schema for any supported value that can populate a table cell. */
+const cellValueSchema = z.union([jsonCellSchema, fileRefSchema]);
+//#endregion
+//#region ../shared/src/schemas/trace.ts
+/** Schema for the semantic categories used to classify trace spans. */
+const traceSpanKindSchema = z.enum([
+	"eval",
+	"agent",
+	"llm",
+	"tool",
+	"retrieval",
+	"scorer",
+	"checkpoint",
+	"custom"
+]);
+/** Schema for the supported presentation formats of trace attributes. */
+const traceAttributeDisplayFormatSchema = z.enum([
+	"string",
+	"number",
+	"duration",
+	"json"
+]);
+/** Schema for the UI locations where a trace attribute can appear. */
+const traceAttributeDisplayPlacementSchema = z.enum([
+	"tree",
+	"detail",
+	"section"
+]);
+/** Schema for resolved trace display rules sent to the UI. */
+const traceAttributeDisplaySchema = z.object({
+	key: z.string().optional(),
+	path: z.string(),
+	label: z.string().optional(),
+	format: traceAttributeDisplayFormatSchema.optional(),
+	numberFormat: numberDisplayOptionsSchema.optional(),
+	placements: z.array(traceAttributeDisplayPlacementSchema).optional(),
+	scope: z.enum(["self", "subtree"]).optional(),
+	mode: z.enum([
+		"all",
+		"last",
+		"sum"
+	]).optional()
+});
+/** Schema for trace display config after transforms have been resolved. */
+const traceDisplayConfigSchema = z.object({ attributes: z.array(traceAttributeDisplaySchema).optional() });
+/** Schema for authored trace display rules accepted from user config. */
+const traceAttributeDisplayInputSchema = z.object({
+	key: z.string().optional(),
+	path: z.string(),
+	label: z.string().optional(),
+	format: traceAttributeDisplayFormatSchema.optional(),
+	numberFormat: numberDisplayOptionsSchema.optional(),
+	placements: z.array(traceAttributeDisplayPlacementSchema).optional(),
+	scope: z.enum(["self", "subtree"]).optional(),
+	mode: z.enum([
+		"all",
+		"last",
+		"sum"
+	]).optional(),
+	transform: z.custom((value) => value === void 0 || typeof value === "function", { message: "Expected a transform function" }).optional()
+});
+/** Schema for authored trace display config in eval or workspace config. */
+const traceDisplayInputConfigSchema = z.object({ attributes: z.array(traceAttributeDisplayInputSchema).optional() });
+/** Schema for a persisted trace span captured during case execution. */
+const traceSpanSchema = z.object({
+	id: z.string(),
+	parentId: z.string().nullable(),
+	caseId: z.string(),
+	kind: traceSpanKindSchema,
+	name: z.string(),
+	startedAt: z.string(),
+	endedAt: z.string().nullable(),
+	status: z.enum([
+		"running",
+		"ok",
+		"error",
+		"cancelled"
+	]),
+	attributes: z.record(z.string(), z.unknown()).optional(),
+	error: z.object({
+		name: z.string().optional(),
+		message: z.string(),
+		stack: z.string().optional()
+	}).optional()
+});
+//#endregion
+//#region ../shared/src/schemas/chart.ts
+/** Chart type rendered for a single eval history chart. */
+const evalChartTypeSchema = z.enum([
+	"area",
+	"line",
+	"bar"
+]);
+/**
+* Run-level metric sourced from the aggregated `RunSummary` for a run, rather
+* than from a per-case column.
+*/
+const evalChartBuiltinMetricSchema = z.enum(["passRate", "durationMs"]);
+/** Reducer applied to a numeric column across all cases of a single run. */
+const evalChartAggregateSchema = z.enum([
+	"avg",
+	"sum",
+	"min",
+	"max",
+	"latest",
+	"passThresholdRate"
+]);
+/**
+* Semantic color token resolved to a theme color by the web UI. The SDK does
+* not emit raw hex so authored evals stay decoupled from the web theme.
+*/
+const evalChartColorSchema = z.enum([
+	"accent",
+	"accentDim",
+	"success",
+	"error",
+	"warning",
+	"textMuted"
+]);
+/** Y-axis placement for a plotted series on a dual-axis chart. */
+const evalChartAxisSchema = z.enum(["left", "right"]);
+/**
+* One plotted series on an eval history chart. `builtin` metrics come from the
+* per-run `RunSummary`; `column` metrics aggregate a per-case score or
+* `setEvalOutput` column across the run using `aggregate`.
+*/
+const evalChartMetricSchema = z.discriminatedUnion("source", [z.object({
+	source: z.literal("builtin"),
+	metric: evalChartBuiltinMetricSchema,
+	label: z.string().optional(),
+	color: evalChartColorSchema.optional(),
+	axis: evalChartAxisSchema.optional()
+}), z.object({
+	source: z.literal("column"),
+	/** Matches a declared score key or a `setEvalOutput` key on the eval. */
+	key: z.string().min(1),
+	aggregate: evalChartAggregateSchema,
+	label: z.string().optional(),
+	color: evalChartColorSchema.optional(),
+	axis: evalChartAxisSchema.optional()
+})]);
+/** Extra field rendered only in the tooltip, not plotted as a series. */
+const evalChartTooltipExtraSchema = z.discriminatedUnion("source", [z.object({
+	source: z.literal("builtin"),
+	metric: evalChartBuiltinMetricSchema,
+	label: z.string().optional()
+}), z.object({
+	source: z.literal("column"),
+	key: z.string().min(1),
+	aggregate: evalChartAggregateSchema,
+	label: z.string().optional()
+})]);
+/**
+* Authored configuration for one eval history chart rendered in `EvalCard`.
+* Authors declare a list of these via `EvalDefinition.charts` — the UI renders
+* each entry as its own chart frame, stacked in authoring order.
+*/
+const evalChartConfigSchema = z.object({
+	/** Optional heading shown above the chart frame in the UI. */
+	heading: z.string().optional(),
+	type: evalChartTypeSchema,
+	/** At least one series must be declared. */
+	metrics: z.array(evalChartMetricSchema).min(1),
+	/**
+	* Per-axis Y domain. Omit either side for automatic scaling. When unset the
+	* chart auto-scales — there is no implicit `[0, 1]` clamp.
+	*/
+	yDomain: z.object({
+		left: z.object({
+			min: z.number().optional(),
+			max: z.number().optional()
+		}).optional(),
+		right: z.object({
+			min: z.number().optional(),
+			max: z.number().optional()
+		}).optional()
+	}).optional(),
+	tooltipExtras: z.array(evalChartTooltipExtraSchema).optional()
+});
+/**
+* Ordered list of history charts rendered for an eval. Opt-in: when omitted or
+* empty, the UI renders no history chart at all.
+*/
+const evalChartsConfigSchema = z.array(evalChartConfigSchema);
+//#endregion
+//#region ../shared/src/schemas/eval.ts
+/** Freshness signal derived from the latest relevant run plus git state. */
+const evalFreshnessStatusSchema = z.enum([
+	"fresh",
+	"stale",
+	"outdated"
+]);
+/** Reducer used to collapse a column's per-case values into a single stat. */
+const evalStatAggregateSchema = z.enum([
+	"avg",
+	"min",
+	"max",
+	"sum",
+	"last"
+]);
+/**
+* One entry in the EvalCard stats row. Built-in kinds use latest run totals;
+* `column` aggregates a score or numeric output column across the latest run.
+*/
+const evalStatItemSchema = z.discriminatedUnion("kind", [
+	z.object({ kind: z.literal("cases") }),
+	z.object({
+		kind: z.literal("passRate"),
+		accent: z.boolean().optional()
+	}),
+	z.object({ kind: z.literal("duration") }),
+	z.object({
+		kind: z.literal("column"),
+		key: z.string(),
+		label: z.string().optional(),
+		aggregate: evalStatAggregateSchema,
+		format: columnFormatSchema.optional(),
+		accent: z.boolean().optional()
+	})
+]);
+/** Ordered list of stats rendered in the EvalCard stats row. */
+const evalStatsConfigSchema = z.array(evalStatItemSchema);
+/** Schema summarizing a discovered eval for list and overview screens. */
+const evalSummarySchema = z.object({
+	id: z.string(),
+	title: z.string().optional(),
+	/** Eval file path relative to the active workspace root. */
+	filePath: z.string(),
+	/** Indicates the eval file changed since the latest passing result. */
+	stale: z.boolean(),
+	/** Indicates the latest comparable run is from an older commit and too old. */
+	outdated: z.boolean(),
+	/** Latest derived freshness signal for this eval. */
+	freshnessStatus: evalFreshnessStatusSchema,
+	/** Timestamp for the latest run considered when deriving freshness. */
+	latestRunAt: z.string().nullable(),
+	/** Commit SHA recorded on the latest run considered for freshness. */
+	latestRunCommitSha: z.string().nullable(),
+	/** Current workspace commit SHA when the summary was requested. */
+	currentCommitSha: z.string().nullable(),
+	columnDefs: z.array(columnDefSchema),
+	caseCount: z.number().nullable(),
+	lastRunStatus: z.enum([
+		"pass",
+		"fail",
+		"error",
+		"running",
+		"cancelled",
+		"unscored"
+	]).nullable(),
+	/**
+	* Optional per-eval stats row configuration for the EvalCard. Opt-in: when
+	* omitted or empty, the UI renders no stats row at all.
+	*/
+	stats: evalStatsConfigSchema.optional(),
+	/**
+	* Ordered per-eval history chart configuration for the EvalCard. Opt-in:
+	* when omitted or empty, the UI renders no history chart at all.
+	*/
+	charts: evalChartsConfigSchema.optional()
+});
+/** Schema for one case row in an eval run result table. */
+const caseRowSchema = z.object({
+	caseId: z.string(),
+	evalId: z.string(),
+	status: z.enum([
+		"pending",
+		"running",
+		"pass",
+		"fail",
+		"error",
+		"cancelled"
+	]),
+	latencyMs: z.number().nullable(),
+	costUsd: z.number().nullable().optional(),
+	columns: z.record(z.string(), cellValueSchema),
+	/** Winning trial index for the persisted case result. */
+	trial: z.number()
+});
+/** Structured assertion failure metadata captured for one case run. */
+const assertionFailureSchema = z.object({
+	/** Human-readable assertion failure message shown in the UI and artifacts. */
+	message: z.string(),
+	/** Stack trace captured from the originating error when available. */
+	stack: z.string().optional()
+});
+const legacyAssertionFailureSchema = z.string().transform((message) => ({ message }));
+/** Trace payload captured while computing one score for a case. */
+const scoreTraceSchema = z.object({
+	trace: z.array(traceSpanSchema),
+	traceDisplay: traceDisplayConfigSchema
+});
+/** Schema for the detailed payload shown when opening a specific case. */
+const caseDetailSchema = z.object({
+	caseId: z.string(),
+	evalId: z.string(),
+	status: z.enum([
+		"pending",
+		"running",
+		"pass",
+		"fail",
+		"error",
+		"cancelled"
+	]),
+	input: z.unknown(),
+	trace: z.array(traceSpanSchema),
+	traceDisplay: traceDisplayConfigSchema,
+	/**
+	* Separate trace payloads emitted by score computation. These are kept out
+	* of `trace` so derive-from-execution metrics do not include judge/scorer
+	* work.
+	*/
+	scoringTraces: z.record(z.string(), scoreTraceSchema).optional(),
+	columns: z.record(z.string(), cellValueSchema),
+	assertionFailures: z.array(z.union([assertionFailureSchema, legacyAssertionFailureSchema])),
+	error: z.object({
+		name: z.string().optional(),
+		message: z.string(),
+		stack: z.string().optional()
+	}).nullable(),
+	/** Winning trial index for the persisted case detail. */
+	trial: z.number()
+});
+//#endregion
+//#region ../shared/src/schemas/cache.ts
+/**
+* Mode that controls how the cache is consulted for a given run.
+*
+* - `use`: read cache on hit, write on miss. Default.
+* - `bypass`: never read, never write.
+* - `refresh`: never read, always write (forces re-execution and overwrites).
+*/
+const cacheModeSchema = z.enum([
+	"use",
+	"bypass",
+	"refresh"
+]);
+/** Options accepted by an `evalTracer.span` call to opt the span into caching. */
+const spanCacheOptionsSchema = z.object({
+	/** Arbitrary JSON-safe value used to derive the cache key. */
+	key: z.unknown(),
+	/** Override the default namespace (`${evalId}__${spanName}`). */
+	namespace: z.string().optional()
+});
+/** Summary of a single persisted cache entry, used by list/delete endpoints. */
+const cacheListItemSchema = z.object({
+	key: z.string(),
+	namespace: z.string(),
+	spanName: z.string(),
+	spanKind: traceSpanKindSchema,
+	storedAt: z.string(),
+	codeFingerprint: z.string(),
+	sizeBytes: z.number()
+});
+/** Zod schema for `SerializedCacheSpan`, defined lazily for recursion. */
+const serializedCacheSpanSchema = z.object({
+	kind: traceSpanKindSchema,
+	name: z.string(),
+	attributes: z.record(z.string(), z.unknown()).optional(),
+	status: z.enum([
+		"running",
+		"ok",
+		"error",
+		"cancelled"
+	]),
+	error: z.object({
+		name: z.string().optional(),
+		message: z.string(),
+		stack: z.string().optional()
+	}).optional()
+}).extend({ children: z.lazy(() => z.array(serializedCacheSpanSchema)) });
+/**
+* One captured operation performed while a cached span's body executed.
+*
+* Operations are replayed in order against a fresh scope on cache hit to
+* reproduce the observable effects of the original run.
+*/
+const cacheRecordingOpSchema = z.discriminatedUnion("kind", [
+	z.object({
+		kind: z.literal("setOutput"),
+		key: z.string(),
+		value: z.unknown()
+	}),
+	z.object({
+		kind: z.literal("incrementOutput"),
+		key: z.string(),
+		delta: z.number()
+	}),
+	z.object({
+		kind: z.literal("checkpoint"),
+		name: z.string(),
+		data: z.unknown()
+	}),
+	z.object({
+		kind: z.literal("subSpan"),
+		span: serializedCacheSpanSchema
+	})
+]);
+/** Captured observable effects + return value of a cached span body. */
+const cacheRecordingSchema = z.object({
+	returnValue: z.unknown(),
+	finalAttributes: z.record(z.string(), z.unknown()),
+	ops: z.array(cacheRecordingOpSchema)
+});
+/** Persisted cache file containing metadata and a recording. */
+const cacheEntrySchema = z.object({
+	version: z.literal(1),
+	key: z.string(),
+	namespace: z.string(),
+	spanName: z.string(),
+	spanKind: traceSpanKindSchema,
+	storedAt: z.string(),
+	codeFingerprint: z.string(),
+	recording: cacheRecordingSchema
+});
+//#endregion
+//#region ../shared/src/schemas/config.ts
+/** Strategy used to collapse repeated trials into one stored case result. */
+const trialSelectionModeSchema = z.enum(["lowestScore", "median"]);
+/** Zod schema for validating `agent-evals.config.ts` input. */
+const agentEvalsConfigSchema = z.object({
+	workspaceRoot: z.string().optional(),
+	include: z.array(z.string()),
+	defaultTrials: z.number().optional(),
+	trialSelection: trialSelectionModeSchema.optional(),
+	concurrency: z.number().optional(),
+	staleAfterDays: z.number().optional(),
+	traceDisplay: traceDisplayInputConfigSchema.optional(),
+	cache: z.object({
+		enabled: z.boolean().optional(),
+		dir: z.string().optional()
+	}).optional()
+});
+//#endregion
+//#region ../shared/src/schemas/run.ts
+/** Schema for persisted metadata about a single run invocation. */
+const runManifestSchema = z.object({
+	id: z.string(),
+	/**
+	* Short, human-readable run id (e.g. `r0`, `r1`). Monotonic global counter
+	* assigned at creation; oldest run is `r0`. Legacy persisted runs are
+	* migrated to have a `shortId` on load.
+	*/
+	shortId: z.string(),
+	status: z.enum([
+		"pending",
+		"running",
+		"completed",
+		"cancelled",
+		"error"
+	]),
+	startedAt: z.string(),
+	endedAt: z.string().nullable(),
+	/**
+	* Git commit SHA for the workspace when the run started. Older persisted
+	* runs may not include this field.
+	*/
+	commitSha: z.string().nullable().optional().default(null),
+	/**
+	* Eval-file fingerprints captured for this run, keyed by eval id. Older
+	* persisted runs may not include this field.
+	*/
+	evalSourceFingerprints: z.record(z.string(), z.string()).optional().default({}),
+	target: z.object({
+		mode: z.enum([
+			"all",
+			"evalIds",
+			"caseIds"
+		]),
+		evalIds: z.array(z.string()).optional(),
+		caseIds: z.array(z.string()).optional()
+	}),
+	/** Number of trial attempts executed for each case in this run. */
+	trials: z.number(),
+	/**
+	* Strategy used to collapse repeated trials into the single persisted case
+	* result for this run. Older persisted runs may not include this field.
+	*/
+	trialSelection: trialSelectionModeSchema.optional().default("lowestScore"),
+	/** Cache mode used for this run. Defaults to `use` when absent. */
+	cacheMode: cacheModeSchema.optional()
+});
+/** Schema for aggregate metrics computed over a completed or active run. */
+const runSummarySchema = z.object({
+	runId: z.string(),
+	status: z.enum([
+		"pending",
+		"running",
+		"completed",
+		"cancelled",
+		"error"
+	]),
+	totalCases: z.number(),
+	passedCases: z.number(),
+	failedCases: z.number(),
+	errorCases: z.number(),
+	cancelledCases: z.number(),
+	totalDurationMs: z.number().nullable(),
+	errorMessage: z.string().nullable().default(null)
+});
+//#endregion
+//#region ../shared/src/status.ts
+function deriveLifecycleStatus(lifecycleStatus) {
+	if (lifecycleStatus === "pending" || lifecycleStatus === "running" || lifecycleStatus === "cancelled" || lifecycleStatus === "error") return lifecycleStatus;
+	return null;
+}
+/**
+* Derive an aggregate status from child statuses, optionally allowing a raw run
+* lifecycle status to override active terminal states such as `running`,
+* `cancelled`, and `error`.
+*/
+function deriveStatusFromChildStatuses(params) {
+	const lifecycle = deriveLifecycleStatus(params.lifecycleStatus);
+	if (lifecycle !== null) return lifecycle;
+	let hasPass = false;
+	let hasPending = false;
+	let hasRunning = false;
+	let hasCancelled = false;
+	let hasError = false;
+	let hasFail = false;
+	for (const status of params.statuses) {
+		if (status === void 0 || status === null) continue;
+		if (status === "running") hasRunning = true;
+		else if (status === "error") hasError = true;
+		else if (status === "fail") hasFail = true;
+		else if (status === "cancelled") hasCancelled = true;
+		else if (status === "pass") hasPass = true;
+		else hasPending = true;
+	}
+	if (hasRunning) return "running";
+	if (hasError) return "error";
+	if (hasFail) return "fail";
+	if (hasCancelled) return "cancelled";
+	if (hasPending || !hasPass) return "pending";
+	return "pass";
+}
+/**
+* Derive an aggregate status from a scoped set of case rows.
+*
+* Pass `lifecycleStatus` only when the parent scope's raw run lifecycle should
+* override the derived child result, such as for a whole-run display.
+*/
+function deriveStatusFromCaseRows(params) {
+	return deriveStatusFromChildStatuses({
+		statuses: Array.from(params.caseRows, (caseRow) => caseRow.status),
+		lifecycleStatus: params.lifecycleStatus
+	});
+}
+/**
+* Derive counts, aggregate metrics, and display status from a scoped set of
+* case rows.
+*/
+function deriveScopedSummaryFromCases(params) {
+	const caseRows = [...params.caseRows];
+	let passedCases = 0;
+	let failedCases = 0;
+	let errorCases = 0;
+	let cancelledCases = 0;
+	let pendingCases = 0;
+	let runningCases = 0;
+	let totalDurationMs = 0;
+	let hasDuration = false;
+	for (const caseRow of caseRows) {
+		if (caseRow.status === "pass") passedCases += 1;
+		else if (caseRow.status === "fail") failedCases += 1;
+		else if (caseRow.status === "error") errorCases += 1;
+		else if (caseRow.status === "cancelled") cancelledCases += 1;
+		else if (caseRow.status === "running") runningCases += 1;
+		else pendingCases += 1;
+		if (caseRow.latencyMs !== null) {
+			totalDurationMs += caseRow.latencyMs;
+			hasDuration = true;
+		}
+	}
+	return {
+		status: deriveStatusFromCaseRows({
+			caseRows,
+			lifecycleStatus: params.lifecycleStatus
+		}),
+		totalCases: caseRows.length,
+		passedCases,
+		failedCases,
+		errorCases,
+		cancelledCases,
+		pendingCases,
+		runningCases,
+		totalDurationMs: hasDuration ? totalDurationMs : null
+	};
+}
+//#endregion
+//#region ../shared/src/evalStatus.ts
+/**
+* Derive the user-facing eval status from the raw latest run result plus
+* freshness state.
+*/
+function getEvalDisplayStatus(params) {
+	const { stale, outdated, lastRunStatus, isRunning = false } = params;
+	if (isRunning || lastRunStatus === "running") return "running";
+	if (lastRunStatus === "pass") {
+		if (stale) return "stale";
+		if (outdated) return "outdated";
+	}
+	return lastRunStatus ?? "pending";
+}
+//#endregion
+//#region ../shared/src/evalTitle.ts
+function humanizeEvalId(id) {
+	const normalized = id.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/[-_\s]+/g, " ").trim();
+	if (normalized.length === 0) return id;
+	return normalized.split(" ").map((segment) => {
+		const firstChar = segment.slice(0, 1);
+		const remainder = segment.slice(1);
+		return `${firstChar.toUpperCase()}${remainder}`;
+	}).join(" ");
+}
+/**
+* Resolve the display title for an eval.
+*
+* Returns the authored `title` when present; otherwise derives a human-readable
+* label from the stable eval `id` so display surfaces can avoid repeating both
+* fields in common cases.
+*/
+function getEvalTitle(evalLike) {
+	if (evalLike.title !== void 0) return evalLike.title;
+	return humanizeEvalId(evalLike.id);
+}
+z.enum([
+	"discovery.updated",
+	"run.started",
+	"run.summary",
+	"case.started",
+	"case.updated",
+	"case.finished",
+	"trace.span",
+	"run.finished",
+	"run.cancelled",
+	"run.error"
+]);
+/** Schema for the SSE envelope used to stream run updates to clients. */
+const sseEnvelopeSchema = z.object({
+	type: z.string(),
+	runId: z.string().optional(),
+	timestamp: z.string(),
+	payload: z.unknown()
+});
+//#endregion
+//#region ../shared/src/schemas/api.ts
+/** Schema for the API request that starts a new eval run. */
+const createRunRequestSchema = z.object({
+	target: z.object({
+		mode: z.enum([
+			"all",
+			"evalIds",
+			"caseIds"
+		]),
+		evalIds: z.array(z.string()).optional(),
+		caseIds: z.array(z.string()).optional()
+	}),
+	trials: z.number().min(1),
+	/**
+	* Optional cache controls for the run. When omitted, the cache is used in
+	* its default read-through / write-on-miss mode.
+	*/
+	cache: z.object({ mode: cacheModeSchema.default("use") }).optional()
+});
+/** Schema for updating a UI-authored manual score on one persisted case. */
+const updateManualScoreRequestSchema = z.object({ value: z.number().min(0).max(1).nullable() });
+//#endregion
+//#region ../runner/src/cacheStore.ts
+/**
+* Create a filesystem-backed cache adapter rooted at `<workspaceRoot>/<dir>`.
+*
+* Writes use `<name>.tmp` + atomic `rename` to avoid partial reads under
+* concurrent access.
+*/
+function createFsCacheStore(options) {
+	const cacheDir = resolve(options.workspaceRoot, options.dir ?? ".agent-evals/cache");
+	return {
+		dir() {
+			return cacheDir;
+		},
+		async lookup(namespace, keyHash) {
+			const filePath = entryPath(cacheDir, namespace, keyHash);
+			if (!existsSync(filePath)) return null;
+			const json = safeJsonParse(await readFile(filePath, "utf-8"));
+			if (json === null) return null;
+			const parsed = cacheEntrySchema.safeParse(json);
+			if (!parsed.success) return null;
+			return parsed.data;
+		},
+		async write(entry) {
+			const filePath = entryPath(cacheDir, entry.namespace, entry.key);
+			await mkdir(dirname(filePath), { recursive: true });
+			const tmpPath = `${filePath}.${process.pid.toString()}.tmp`;
+			await writeFile(tmpPath, JSON.stringify(entry));
+			await rename(tmpPath, filePath);
+		},
+		async list() {
+			if (!existsSync(cacheDir)) return [];
+			const namespaces = await readdir(cacheDir);
+			const items = [];
+			for (const namespace of namespaces) {
+				const nsPath = join(cacheDir, namespace);
+				if (!(await stat(nsPath)).isDirectory()) continue;
+				const files = await readdir(nsPath);
+				for (const fileName of files) {
+					if (!fileName.endsWith(".json")) continue;
+					const filePath = join(nsPath, fileName);
+					const json = safeJsonParse(await readFile(filePath, "utf-8"));
+					if (json === null) continue;
+					const parsed = cacheEntrySchema.safeParse(json);
+					if (!parsed.success) continue;
+					const fileStat = await stat(filePath);
+					items.push({
+						key: parsed.data.key,
+						namespace: parsed.data.namespace,
+						spanName: parsed.data.spanName,
+						spanKind: parsed.data.spanKind,
+						storedAt: parsed.data.storedAt,
+						codeFingerprint: parsed.data.codeFingerprint,
+						sizeBytes: fileStat.size
+					});
+				}
+			}
+			items.sort((a, b) => a.storedAt < b.storedAt ? 1 : -1);
+			return items;
+		},
+		async clear(filter) {
+			if (!existsSync(cacheDir)) return;
+			if (!filter || filter.namespace === void 0 && filter.key === void 0) {
+				await rm(cacheDir, {
+					recursive: true,
+					force: true
+				});
+				return;
+			}
+			if (filter.namespace !== void 0 && filter.key === void 0) {
+				await rm(join(cacheDir, filter.namespace), {
+					recursive: true,
+					force: true
+				});
+				return;
+			}
+			if (filter.namespace !== void 0 && filter.key !== void 0) {
+				await rm(entryPath(cacheDir, filter.namespace, filter.key), { force: true });
+				return;
+			}
+			const namespaces = await readdir(cacheDir);
+			for (const namespace of namespaces) {
+				const filePath = entryPath(cacheDir, namespace, filter.key ?? "");
+				if (existsSync(filePath)) await rm(filePath, { force: true });
+			}
+		}
+	};
+}
+/**
+* Create a write-buffered cache adapter for one trial attempt.
+*
+* Lookups first consult entries written earlier in the same trial, then fall
+* back to the shared backing store. Call `commit()` after selecting the
+* winning trial so only that trial's writes reach the shared cache.
+*/
+function createBufferedCacheStore(backingStore) {
+	const pendingEntries = /* @__PURE__ */ new Map();
+	return {
+		async lookup(namespace, keyHash) {
+			const buffered = pendingEntries.get(toPendingKey(namespace, keyHash));
+			if (buffered !== void 0) return buffered;
+			return backingStore.lookup(namespace, keyHash);
+		},
+		write(entry) {
+			pendingEntries.set(toPendingKey(entry.namespace, entry.key), entry);
+			return Promise.resolve();
+		},
+		async commit() {
+			for (const entry of pendingEntries.values()) await backingStore.write(entry);
+		},
+		getPendingEntries() {
+			return [...pendingEntries.values()];
+		}
+	};
+}
+function entryPath(cacheDir, namespace, keyHash) {
+	return join(cacheDir, sanitizeSegment$1(namespace), `${keyHash}.json`);
+}
+function toPendingKey(namespace, keyHash) {
+	return `${namespace}::${keyHash}`;
+}
+function sanitizeSegment$1(segment) {
+	return segment.replace(/[^a-zA-Z0-9_.-]/g, "_");
+}
+function safeJsonParse(text) {
+	const parsed = resultify(() => JSON.parse(text));
+	if (parsed.error) return null;
+	return parsed.value;
+}
+//#endregion
+//#region ../runner/src/chartValidation.ts
+function isValidColumnMetric(metric, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(metric.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart metric references unknown column "${metric.key}" — dropped`);
+		return false;
+	}
+	if (metric.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart metric "${metric.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function isValidTooltipExtra(extra, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(extra.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart tooltip extra references unknown column "${extra.key}" — dropped`);
+		return false;
+	}
+	if (extra.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart tooltip extra "${extra.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function sanitizeChart(chart, columnsByKey, evalId, warnings) {
+	const metrics = chart.metrics.filter((metric) => {
+		if (metric.source === "builtin") return true;
+		return isValidColumnMetric(metric, columnsByKey, evalId, warnings);
+	});
+	if (metrics.length === 0) {
+		warnings.push(`[${evalId}] chart had no valid metrics after validation — chart dropped`);
+		return null;
+	}
+	const tooltipExtras = chart.tooltipExtras?.filter((extra) => {
+		if (extra.source === "builtin") return true;
+		return isValidTooltipExtra(extra, columnsByKey, evalId, warnings);
+	});
+	return {
+		...chart,
+		metrics,
+		tooltipExtras: tooltipExtras?.length ? tooltipExtras : void 0
+	};
+}
+/**
+* Validate and sanitize an authored `charts` config against the eval's
+* declared columns. Drops metrics/extras that reference unknown columns or
+* misuse `passThresholdRate`, and drops entire charts whose metrics are all
+* invalid. Returns `charts: undefined` when nothing valid remains so the UI
+* falls back to rendering no chart (matching the opt-in default).
+*/
+function validateCharts(params) {
+	const { charts, columnDefs, evalId } = params;
+	if (!charts || charts.length === 0) return {
+		charts: void 0,
+		warnings: []
+	};
+	const columnsByKey = new Map(columnDefs.map((def) => [def.key, def]));
+	const warnings = [];
+	const sanitized = [];
+	for (const chart of charts) {
+		const result = sanitizeChart(chart, columnsByKey, evalId, warnings);
+		if (result) sanitized.push(result);
+	}
+	return {
+		charts: sanitized.length > 0 ? sanitized : void 0,
+		warnings
+	};
+}
+//#endregion
+//#region ../runner/src/columnBuilder.ts
+/**
+* Normalize a user-provided score definition (either a function or an
+* object literal with `compute`/`passThreshold`/`label`) to a common
+* shape used internally.
+*/
+function normalizeScoreDef(def) {
+	if (typeof def === "function") return {
+		compute: def,
+		passThreshold: void 0,
+		label: void 0
+	};
+	return {
+		compute: def.compute,
+		passThreshold: def.passThreshold,
+		label: def.label
+	};
+}
+function getScoreOverride(def) {
+	if (def === void 0 || typeof def === "function") return void 0;
+	return {
+		label: def.label,
+		format: def.format,
+		numberFormat: def.numberFormat,
+		hideInTable: def.hideInTable,
+		sortable: def.sortable,
+		align: def.align,
+		maxStars: def.maxStars
+	};
+}
+function mergeOverrides(base, override) {
+	if (base === void 0) return override;
+	if (override === void 0) return base;
+	return {
+		label: override.label ?? base.label,
+		format: override.format ?? base.format,
+		numberFormat: override.numberFormat ?? base.numberFormat,
+		hideInTable: override.hideInTable ?? base.hideInTable,
+		sortable: override.sortable ?? base.sortable,
+		align: override.align ?? base.align,
+		maxStars: override.maxStars ?? base.maxStars
+	};
+}
+/**
+* Populate `target` with `ColumnDef` entries for any keys in `columns`
+* that aren't already present, applying user-supplied `overrides` and
+* flagging score columns declared via `scores`.
+*/
+function mergeColumnDefs(target, columns, overrides, scores, manualScores) {
+	const scoreKeys = new Set(Object.keys(scores ?? {}));
+	const manualScoreKeys = new Set(Object.keys(manualScores ?? {}));
+	const overrideMap = overrides ?? {};
+	for (const [key, value] of Object.entries(columns)) {
+		if (target.has(key)) continue;
+		const override = mergeOverrides(getScoreOverride(scores?.[key]) ?? manualScores?.[key], overrideMap[key]);
+		const isScore = scoreKeys.has(key) || manualScoreKeys.has(key);
+		target.set(key, createColumnDef({
+			key,
+			override,
+			scoreDef: scores?.[key],
+			manualScoreDef: manualScores?.[key],
+			inferredKind: isScore ? "number" : inferKind(value),
+			isScore,
+			isManualScore: manualScoreKeys.has(key)
+		}));
+	}
+}
+/**
+* Build the column definitions declared directly on an eval before any runtime
+* output values exist. This lets discovery metadata describe authored rich
+* output columns even for runs created by another process.
+*/
+function buildDeclaredColumnDefs(overrides, scores, manualScores) {
+	const declaredDefs = /* @__PURE__ */ new Map();
+	for (const [key, override] of Object.entries(overrides ?? {})) {
+		const isScore = scores?.[key] !== void 0 || manualScores?.[key] !== void 0;
+		const mergedOverride = mergeOverrides(getScoreOverride(scores?.[key]) ?? manualScores?.[key], override);
+		declaredDefs.set(key, createColumnDef({
+			key,
+			override: mergedOverride,
+			scoreDef: scores?.[key],
+			manualScoreDef: manualScores?.[key],
+			inferredKind: inferKindFromFormat(mergedOverride?.format) ?? (mergedOverride?.numberFormat === void 0 ? void 0 : "number"),
+			isScore,
+			isManualScore: manualScores?.[key] !== void 0
+		}));
+	}
+	for (const [key, scoreDef] of Object.entries(scores ?? {})) {
+		if (declaredDefs.has(key)) continue;
+		declaredDefs.set(key, createColumnDef({
+			key,
+			override: getScoreOverride(scoreDef),
+			scoreDef,
+			inferredKind: "number",
+			isScore: true,
+			isManualScore: false
+		}));
+	}
+	for (const [key, manualScoreDef] of Object.entries(manualScores ?? {})) {
+		if (declaredDefs.has(key)) continue;
+		declaredDefs.set(key, createColumnDef({
+			key,
+			override: manualScoreDef,
+			manualScoreDef,
+			inferredKind: "number",
+			isScore: true,
+			isManualScore: true
+		}));
+	}
+	return [...declaredDefs.values()];
+}
+/** Infer a `ColumnKind` from a runtime value when no override is set. */
+function inferKind(value) {
+	if (typeof value === "number") return "number";
+	if (typeof value === "boolean") return "boolean";
+	return "string";
+}
+/**
+* Coerce an arbitrary runtime value into a serializable `CellValue`.
+* Non-primitive values fall back to `JSON.stringify`.
+*/
+function toCellValue(value, override = void 0) {
+	if (value === null) return null;
+	if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
+	if (value === void 0) return void 0;
+	if (override?.format === "image" || override?.format === "audio" || override?.format === "video" || override?.format === "file") {
+		const parsed = fileRefSchema.safeParse(value);
+		if (parsed.success) return parsed.data;
+	}
+	if (override?.format === "json") {
+		const parsed = jsonCellSchema.safeParse(value);
+		if (parsed.success) return parsed.data;
+	}
+	return JSON.stringify(value);
+}
+function inferKindFromFormat(format) {
+	if (format === "boolean") return "boolean";
+	if (format === "duration" || format === "percent" || format === "number" || format === "passFail" || format === "stars") return "number";
+	if (format === void 0) return void 0;
+	return "string";
+}
+function createColumnDef(params) {
+	const { key, override, scoreDef, manualScoreDef, inferredKind, isScore, isManualScore } = params;
+	const kind = inferredKind ?? (isScore ? "number" : "string");
+	const def = {
+		key,
+		label: override?.label ?? key,
+		kind
+	};
+	if (override?.format !== void 0) def.format = override.format;
+	if (override?.numberFormat !== void 0) def.numberFormat = override.numberFormat;
+	if (override?.maxStars !== void 0) def.maxStars = override.maxStars;
+	if (override?.hideInTable !== void 0) def.hideInTable = override.hideInTable;
+	if (override?.sortable !== void 0) def.sortable = override.sortable;
+	if (override?.align !== void 0) def.align = override.align;
+	if (!isScore) return def;
+	def.isScore = true;
+	if (isManualScore) {
+		def.isManualScore = true;
+		if (manualScoreDef?.passThreshold !== void 0) def.passThreshold = manualScoreDef.passThreshold;
+		return def;
+	}
+	if (typeof scoreDef === "function" || scoreDef === void 0) return def;
+	if (scoreDef.passThreshold !== void 0) def.passThreshold = scoreDef.passThreshold;
+	if (scoreDef.label !== void 0 && override?.label === void 0) def.label = scoreDef.label;
+	return def;
+}
+//#endregion
+//#region ../runner/src/config.ts
+const configModuleSchema = z.object({
+	default: agentEvalsConfigSchema.optional(),
+	config: agentEvalsConfigSchema.optional()
+});
+const defaultConfig = {
+	include: ["**/*.eval.ts"],
+	defaultTrials: 1,
+	trialSelection: "lowestScore",
+	concurrency: 2,
+	staleAfterDays: 14,
+	traceDisplay: { attributes: [{
+		path: "input",
+		label: "Input",
+		format: "json",
+		placements: ["section"]
+	}, {
+		path: "output",
+		label: "Output",
+		format: "json",
+		placements: ["section"]
+	}] }
+};
+async function loadConfig() {
+	const configPath = resolve(process.cwd(), "agent-evals.config.ts");
+	if (!existsSync(configPath)) return defaultConfig;
+	try {
+		const imported = await import(pathToFileURL(configPath).href);
+		const configModule = configModuleSchema.parse(imported);
+		const userConfig = configModule.default ?? configModule.config;
+		if (!userConfig) return defaultConfig;
+		return {
+			...defaultConfig,
+			...userConfig
+		};
+	} catch (error) {
+		console.error("Failed to load agent-evals.config.ts:", error);
+		return defaultConfig;
+	}
+}
+//#endregion
+//#region ../runner/src/discovery.ts
+const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
+const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
+function parseEvalMetas(filePath, content) {
+	const metas = [];
+	let searchIndex = 0;
+	while (searchIndex < content.length) {
+		const defineEvalIndex = content.indexOf("defineEval", searchIndex);
+		if (defineEvalIndex === -1) break;
+		const extracted = extractDefineEvalObject(content, defineEvalIndex);
+		if (!extracted) {
+			searchIndex = defineEvalIndex + 10;
+			continue;
+		}
+		const id = evalIdMatchRegex.exec(extracted.objectText)?.[1];
+		if (id !== void 0) {
+			const result = {
+				filePath,
+				id
+			};
+			const title = evalTitleMatchRegex.exec(extracted.objectText)?.[1];
+			if (title !== void 0) result.title = title;
+			metas.push(result);
+		}
+		searchIndex = extracted.nextIndex;
+	}
+	return metas;
+}
+function extractDefineEvalObject(content, defineEvalIndex) {
+	const openParenIndex = content.indexOf("(", defineEvalIndex);
+	if (openParenIndex === -1) return void 0;
+	const objectStartIndex = content.indexOf("{", openParenIndex);
+	if (objectStartIndex === -1) return void 0;
+	let depth = 0;
+	let quote;
+	let inBlockComment = false;
+	let inLineComment = false;
+	let isEscaped = false;
+	for (let index = objectStartIndex; index < content.length; index++) {
+		const currentChar = content[index];
+		const nextChar = content[index + 1];
+		if (inLineComment) {
+			if (currentChar === "\n") inLineComment = false;
+			continue;
+		}
+		if (inBlockComment) {
+			if (currentChar === "*" && nextChar === "/") {
+				inBlockComment = false;
+				index++;
+			}
+			continue;
+		}
+		if (quote) {
+			if (isEscaped) {
+				isEscaped = false;
+				continue;
+			}
+			if (currentChar === "\\") {
+				isEscaped = true;
+				continue;
+			}
+			if (currentChar === quote) quote = void 0;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "/") {
+			inLineComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "*") {
+			inBlockComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "\"" || currentChar === "'" || currentChar === "`") {
+			quote = currentChar;
+			continue;
+		}
+		if (currentChar === "{") {
+			depth++;
+			continue;
+		}
+		if (currentChar === "}") {
+			depth--;
+			if (depth === 0) return {
+				nextIndex: index + 1,
+				objectText: content.slice(objectStartIndex, index + 1)
+			};
+		}
+	}
+}
+//#endregion
+//#region ../runner/src/evalModuleLoader.ts
+/**
+* Import one eval module with a cache key derived from its current source so
+* repeated discovery and runs observe the latest authored definition.
+*/
+async function loadEvalModule(filePath, sourceFingerprint = void 0) {
+	const moduleUrl = new URL(pathToFileURL(filePath).href);
+	if (sourceFingerprint !== void 0) moduleUrl.searchParams.set("v", sourceFingerprint);
+	await import(moduleUrl.href);
+}
+//#endregion
+//#region ../runner/src/freshness.ts
+/**
+* Derive eval freshness from the latest run, current eval-file fingerprint,
+* current git commit, and an age threshold.
+*/
+function deriveEvalFreshness(params) {
+	const { latestRun, gitState, currentEvalSourceFingerprint, staleAfterDays, now = /* @__PURE__ */ new Date() } = params;
+	const stale = latestRun?.evalSourceFingerprint !== void 0 && latestRun.evalSourceFingerprint !== null && currentEvalSourceFingerprint !== null && currentEvalSourceFingerprint !== latestRun.evalSourceFingerprint;
+	const latestRunCommitSha = latestRun?.commitSha;
+	if (latestRunCommitSha === void 0 || latestRunCommitSha === null) return {
+		freshnessStatus: stale ? "stale" : "fresh",
+		stale,
+		outdated: false
+	};
+	if (gitState.commitSha === null) return {
+		freshnessStatus: stale ? "stale" : "fresh",
+		stale,
+		outdated: false
+	};
+	if (latestRunCommitSha === gitState.commitSha) return {
+		freshnessStatus: stale ? "stale" : "fresh",
+		stale,
+		outdated: false
+	};
+	const latestRunStartedAt = new Date(latestRun?.startedAt ?? "").getTime();
+	if (!Number.isFinite(latestRunStartedAt)) return {
+		freshnessStatus: stale ? "stale" : "fresh",
+		stale,
+		outdated: false
+	};
+	const outdated = now.getTime() - latestRunStartedAt >= staleAfterDays * 24 * 60 * 60 * 1e3;
+	return {
+		freshnessStatus: stale ? "stale" : outdated ? "outdated" : "fresh",
+		stale,
+		outdated
+	};
+}
+/** Return the timestamp used when ordering and displaying a run recency. */
+function getRunFreshnessTimestamp(manifest) {
+	return manifest.endedAt ?? manifest.startedAt;
+}
+//#endregion
+//#region ../runner/src/evalSummaries.ts
+/** Build the API/UI summary payload for one discovered eval. */
+function buildEvalSummary(params) {
+	const { meta, config, gitState, latestRun, lastRunStatus } = params;
+	const { sourceFingerprint, ...summaryMeta } = meta;
+	const freshness = deriveEvalFreshness({
+		latestRun,
+		gitState,
+		currentEvalSourceFingerprint: sourceFingerprint,
+		staleAfterDays: config.staleAfterDays ?? 14
+	});
+	return {
+		...summaryMeta,
+		stale: freshness.stale,
+		outdated: freshness.outdated,
+		freshnessStatus: freshness.freshnessStatus,
+		latestRunAt: latestRun?.startedAt ?? null,
+		latestRunCommitSha: latestRun?.commitSha ?? null,
+		currentCommitSha: gitState.commitSha,
+		lastRunStatus
+	};
+}
+/** Resolve which eval ids a run request should mark as the latest run. */
+function getTargetEvalIds(params) {
+	const { request, sortedEvalIds, knownEvalIds } = params;
+	if (request.target.evalIds && request.target.evalIds.length > 0) return request.target.evalIds.filter((evalId) => knownEvalIds.has(evalId));
+	return sortedEvalIds;
+}
+/** Write one latest-run snapshot to each targeted eval id. */
+function setLatestRunInfoMap(params) {
+	const { latestRunInfoMap, evalIds, info } = params;
+	for (const evalId of evalIds) latestRunInfoMap.set(evalId, info);
+}
+//#endregion
+//#region ../runner/src/gitState.ts
+function runGitCommand(workspaceRoot, args) {
+	const result = spawnSync("git", args, {
+		cwd: workspaceRoot,
+		encoding: "utf8",
+		stdio: [
+			"ignore",
+			"pipe",
+			"ignore"
+		]
+	});
+	return {
+		status: result.status,
+		stdout: result.stdout.trim()
+	};
+}
+/** Read the current git commit for the workspace, if available. */
+function readGitWorktreeState(workspaceRoot) {
+	const insideWorktree = runGitCommand(workspaceRoot, ["rev-parse", "--is-inside-work-tree"]);
+	if (insideWorktree.status !== 0 || insideWorktree.stdout !== "true") return { commitSha: null };
+	const commitResult = runGitCommand(workspaceRoot, ["rev-parse", "HEAD"]);
+	return { commitSha: commitResult.status === 0 ? commitResult.stdout : null };
+}
+//#endregion
+//#region ../runner/src/outputArtifacts.ts
+const mimeTypeExtensionMap = {
+	"application/json": ".json",
+	"application/pdf": ".pdf",
+	"audio/mpeg": ".mp3",
+	"audio/mp4": ".m4a",
+	"audio/wav": ".wav",
+	"image/gif": ".gif",
+	"image/jpeg": ".jpg",
+	"image/png": ".png",
+	"image/svg+xml": ".svg",
+	"image/webp": ".webp",
+	"text/html": ".html",
+	"text/markdown": ".md",
+	"text/plain": ".txt",
+	"video/mp4": ".mp4",
+	"video/webm": ".webm"
+};
+/**
+* Persist a `Blob`/`File` emitted via `setEvalOutput(...)` into the current run's
+* artifact directory and return the resulting run artifact reference.
+*/
+async function persistInlineArtifact({ artifactDir, runId, caseId, outputKey, trial, value }) {
+	await mkdir(artifactDir, { recursive: true });
+	const mimeType = normalizeMimeType(value.type);
+	const fileName = getArtifactFileName({
+		outputKey,
+		mimeType,
+		value
+	});
+	const artifactId = [
+		sanitizeSegment(runId),
+		sanitizeSegment(caseId),
+		`t${String(trial)}`,
+		sanitizeSegment(outputKey),
+		sanitizeFileName(fileName)
+	].join("__");
+	await writeFile(join(artifactDir, artifactId), new Uint8Array(await value.arrayBuffer()));
+	return {
+		source: "run",
+		artifactId,
+		mimeType,
+		fileName
+	};
+}
+/** Resolve a persisted run artifact path from its artifact id. */
+function resolveArtifactPath(runsDir, artifactId) {
+	const [runId] = artifactId.split("__", 1);
+	if (!runId) return void 0;
+	return join(runsDir, runId, "artifacts", artifactId);
+}
+function normalizeMimeType(value) {
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : "application/octet-stream";
+}
+function getArtifactFileName(params) {
+	const { outputKey, mimeType, value } = params;
+	if (isFile(value) && value.name.trim().length > 0) return value.name.trim();
+	const extension = getExtensionForMimeType(mimeType);
+	return extension.length > 0 ? `${sanitizeSegment(outputKey)}${extension}` : sanitizeSegment(outputKey);
+}
+function getExtensionForMimeType(mimeType) {
+	const exactMatch = mimeTypeExtensionMap[mimeType];
+	if (exactMatch) return exactMatch;
+	const subtype = mimeType.split("/")[1];
+	if (subtype === void 0 || subtype.length === 0) return "";
+	const withoutSuffix = subtype.split("+")[0] ?? subtype;
+	return withoutSuffix.length > 0 ? `.${withoutSuffix}` : "";
+}
+function sanitizeSegment(value) {
+	const normalized = value.trim().replaceAll(/[^A-Za-z0-9._-]+/g, "-");
+	return normalized.length > 0 ? normalized : "artifact";
+}
+function sanitizeFileName(value) {
+	const normalized = sanitizeSegment(value);
+	const extension = extname(normalized);
+	if (extension.length === 0) return normalized;
+	return `${normalized.slice(0, -extension.length).replaceAll(".", "-")}${extension}`;
+}
+function isFile(value) {
+	return value instanceof File;
+}
+//#endregion
+//#region ../runner/src/runMaintenance.ts
+async function persistRunState(runState) {
+	await writeFile(join(runState.runDir, "summary.json"), JSON.stringify(runState.summary, null, 2));
+	await writeFile(join(runState.runDir, "run.json"), JSON.stringify(runState.manifest, null, 2));
+	const casesJsonl = runState.cases.map((c) => JSON.stringify(c)).join("\n");
+	await writeFile(join(runState.runDir, "cases.jsonl"), casesJsonl);
+}
+/**
+* Recompute a persisted case's status after score definitions changed.
+*
+* Pass/fail gates are per-score: a case fails when any score with a declared
+* `passThreshold` reports a numeric value below that threshold. Scores
+* without a threshold are informational and never gate. Cancelled and
+* errored cases retain their terminal status.
+*/
+function recomputePersistedCaseStatus(caseRow, caseDetail, scoreThresholds) {
+	if (caseRow.status === "cancelled") return "cancelled";
+	if (caseDetail?.error !== null && caseDetail?.error !== void 0) return "error";
+	if ((caseDetail?.assertionFailures.length ?? 0) > 0) return "fail";
+	for (const [key, passThreshold] of scoreThresholds) {
+		const rawValue = caseRow.columns[key] ?? caseDetail?.columns[key];
+		if (typeof rawValue !== "number") continue;
+		if (rawValue < passThreshold) return "fail";
+	}
+	return caseRow.status === "error" ? "error" : "pass";
+}
+function runTouchesEval(params) {
+	if (params.caseRows.some((caseRow) => caseRow.evalId === params.evalId)) return true;
+	if (params.target.mode === "all") return params.evalExists;
+	if (params.target.mode === "evalIds") return params.target.evalIds?.includes(params.evalId) ?? false;
+	return false;
+}
+async function recomputeEvalStatusesInRuns(params) {
+	let updatedRuns = 0;
+	for (const run of params.runs) {
+		if (!runTouchesEval({
+			target: run.manifest.target,
+			caseRows: run.cases,
+			evalId: params.evalId,
+			evalExists: params.evalExists
+		})) continue;
+		if (run.manifest.status === "running") continue;
+		let changed = false;
+		for (const caseRow of run.cases) {
+			if (caseRow.evalId !== params.evalId) continue;
+			const caseDetail = run.caseDetails.get(caseRow.caseId);
+			const nextStatus = recomputePersistedCaseStatus(caseRow, caseDetail, params.scoreThresholds);
+			if (caseRow.status === nextStatus) continue;
+			caseRow.status = nextStatus;
+			if (caseDetail) {
+				caseDetail.status = nextStatus;
+				await params.persistCaseDetail(run.runDir, caseDetail);
+			}
+			changed = true;
+		}
+		if (!changed) continue;
+		const derivedSummary = deriveScopedSummaryFromCases({ caseRows: run.cases });
+		run.summary.totalCases = derivedSummary.totalCases;
+		run.summary.passedCases = derivedSummary.passedCases;
+		run.summary.failedCases = derivedSummary.failedCases;
+		run.summary.errorCases = derivedSummary.errorCases;
+		run.summary.cancelledCases = derivedSummary.cancelledCases;
+		await persistRunState(run);
+		updatedRuns += 1;
+	}
+	return updatedRuns;
+}
+//#endregion
+//#region ../runner/src/traceDisplay.ts
+function isRecord$1(value) {
+	return typeof value === "object" && value !== null;
+}
+function getNestedAttribute(value, path) {
+	const parts = path.split(".");
+	let current = value;
+	for (const part of parts) {
+		if (!isRecord$1(current) || !(part in current)) return;
+		current = current[part];
+	}
+	return current;
+}
+function mergeNestedAttribute(value, path, attributeValue) {
+	const root = value === void 0 ? {} : { ...value };
+	const parts = path.split(".");
+	let current = root;
+	for (const [index, part] of parts.entries()) {
+		if (index === parts.length - 1) {
+			current[part] = attributeValue;
+			continue;
+		}
+		const nextValue = current[part];
+		const nextRecord = isRecord$1(nextValue) ? { ...nextValue } : {};
+		current[part] = nextRecord;
+		current = nextRecord;
+	}
+	return root;
+}
+function resolveTracePresentation(spans, globalTraceDisplay, evalTraceDisplay) {
+	const merged = /* @__PURE__ */ new Map();
+	for (const attribute of globalTraceDisplay?.attributes ?? []) merged.set(attribute.key ?? attribute.path, attribute);
+	for (const attribute of evalTraceDisplay?.attributes ?? []) merged.set(attribute.key ?? attribute.path, attribute);
+	const resolvedAttributes = [];
+	const transformedTrace = spans.map((span) => ({
+		...span,
+		attributes: span.attributes === void 0 ? void 0 : { ...span.attributes }
+	}));
+	for (const attribute of merged.values()) {
+		const resolvedPath = attribute.transform ? `__display.${attribute.key ?? attribute.path}` : attribute.path;
+		resolvedAttributes.push({
+			key: attribute.key,
+			path: resolvedPath,
+			label: attribute.label,
+			format: attribute.format,
+			numberFormat: attribute.numberFormat,
+			placements: attribute.placements,
+			scope: attribute.scope,
+			mode: attribute.mode
+		});
+		if (!attribute.transform) continue;
+		for (const span of transformedTrace) {
+			const sourceValue = getNestedAttribute(span.attributes, attribute.path);
+			if (sourceValue === void 0) continue;
+			const transformedValue = attribute.transform({
+				value: sourceValue,
+				span
+			});
+			if (transformedValue === void 0) continue;
+			span.attributes = mergeNestedAttribute(span.attributes, resolvedPath, transformedValue);
+		}
+	}
+	return {
+		trace: transformedTrace,
+		traceDisplay: { attributes: resolvedAttributes }
+	};
+}
+//#endregion
+//#region ../runner/src/runExecution.ts
+function filterEvalCases(cases, evalIds, caseIds, evalId) {
+	if (evalIds && evalIds.length > 0 && !evalIds.includes(evalId)) return [];
+	if (!caseIds || caseIds.length === 0) return cases;
+	const selectedCaseIds = new Set(caseIds);
+	return cases.filter((evalCase) => selectedCaseIds.has(evalCase.id));
+}
+function resolveRunnableEvalCases(params) {
+	const { cases, evalId } = params;
+	if (cases.length > 0) return cases;
+	return [{
+		id: `${evalId}-no-output`,
+		input: {}
+	}];
+}
+async function callWithUnknownResult(fn, args) {
+	return await Reflect.apply(fn, void 0, args);
+}
+async function runCase(params) {
+	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, signal, startTime, cacheAdapter, cacheMode, codeFingerprint, artifactDir, runId } = params;
+	const { scope, error: executeError } = await runInEvalScope(evalCase.id, async () => {
+		await Reflect.apply(evalDef.execute, evalDef, [{
+			input: evalCase.input,
+			signal
+		}]);
+	}, { cacheContext: cacheAdapter ? {
+		adapter: cacheAdapter,
+		mode: cacheMode,
+		evalId,
+		codeFingerprint
+	} : void 0 });
+	const traceTree = buildTraceTree(scope.spans, scope.checkpoints);
+	const nonAssertError = executeError && !(executeError instanceof EvalAssertionError) ? executeError : null;
+	if (executeError instanceof EvalAssertionError && scope.assertionFailures.length === 0) scope.assertionFailures.push(toAssertionFailure(executeError.message, executeError));
+	if (!nonAssertError && evalDef.deriveFromTracing) try {
+		const derived = await callWithUnknownResult(evalDef.deriveFromTracing, [{
+			trace: traceTree,
+			input: evalCase.input,
+			case: evalCase
+		}]);
+		if (!isRecord(derived)) throw new Error("deriveFromTracing must return an object");
+		for (const [key, value] of Object.entries(derived)) if (!(key in scope.outputs)) scope.outputs[key] = value;
+	} catch (e) {
+		const message = `deriveFromTracing threw: ${e instanceof Error ? e.message : String(e)}`;
+		scope.assertionFailures.push(toAssertionFailure(message, e instanceof Error ? e : void 0));
+	}
+	const scoreResults = /* @__PURE__ */ new Map();
+	const scoringTraces = {};
+	if (!nonAssertError && evalDef.scores) for (const [key, def] of Object.entries(evalDef.scores)) {
+		const { compute, passThreshold, label } = normalizeScoreDef(def);
+		const scoreRun = await runInEvalScope(evalCase.id, async () => await callWithUnknownResult(compute, [{
+			input: evalCase.input,
+			outputs: { ...scope.outputs },
+			case: evalCase
+		}]), { cacheContext: cacheAdapter ? {
+			adapter: cacheAdapter,
+			mode: cacheMode,
+			evalId: `${evalId}__score__${key}`,
+			codeFingerprint
+		} : void 0 });
+		const { trace, traceDisplay } = resolveTracePresentation(scoreRun.scope.spans, globalTraceDisplay, evalDef.traceDisplay);
+		if (trace.length > 0) scoringTraces[key] = {
+			trace,
+			traceDisplay
+		};
+		const rawValue = scoreRun.result;
+		if (scoreRun.error) {
+			const message = `score "${key}" threw: ${scoreRun.error.message}`;
+			scope.assertionFailures.push(toAssertionFailure(message, scoreRun.error));
+			scope.outputs[key] = 0;
+			scoreResults.set(key, {
+				value: 0,
+				passThreshold,
+				label
+			});
+			continue;
+		}
+		if (typeof rawValue !== "number") {
+			scope.assertionFailures.push(toAssertionFailure(`score "${key}" must return a number`));
+			scope.outputs[key] = 0;
+			scoreResults.set(key, {
+				value: 0,
+				passThreshold,
+				label
+			});
+			continue;
+		}
+		const value = rawValue;
+		scope.outputs[key] = value;
+		scoreResults.set(key, {
+			value,
+			passThreshold,
+			label
+		});
+	}
+	let passed = scope.assertionFailures.length === 0 && !nonAssertError;
+	if (passed) {
+		for (const [, scoreEntry] of scoreResults) if (scoreEntry.passThreshold !== void 0 && scoreEntry.value < scoreEntry.passThreshold) {
+			passed = false;
+			break;
+		}
+	}
+	const status = nonAssertError ? "error" : passed ? "pass" : "fail";
+	const { trace: displayTrace, traceDisplay } = resolveTracePresentation(scope.spans, globalTraceDisplay, evalDef.traceDisplay);
+	const columns = {};
+	for (const [key, value] of Object.entries(scope.outputs)) {
+		const cell = isBlob(value) ? await persistInlineArtifact({
+			artifactDir,
+			runId,
+			caseId: evalCase.id,
+			outputKey: key,
+			trial,
+			value
+		}) : toCellValue(value, evalDef.columns?.[key]);
+		if (cell !== void 0) columns[key] = cell;
+	}
+	for (const key of Object.keys(evalDef.manualScores ?? {})) columns[key] = null;
+	const errorInfo = nonAssertError ? {
+		name: nonAssertError.name,
+		message: nonAssertError.message,
+		stack: nonAssertError.stack
+	} : null;
+	const caseDetail = {
+		caseId: evalCase.id,
+		evalId,
+		status,
+		input: evalCase.input,
+		trace: displayTrace,
+		traceDisplay,
+		columns,
+		assertionFailures: scope.assertionFailures,
+		error: errorInfo,
+		trial
+	};
+	if (Object.keys(scoringTraces).length > 0) caseDetail.scoringTraces = scoringTraces;
+	return {
+		caseDetail,
+		caseRowUpdate: {
+			status,
+			latencyMs: Date.now() - startTime,
+			columns
+		}
+	};
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null;
+}
+function isBlob(value) {
+	return value instanceof Blob;
+}
+function toAssertionFailure(message, error = void 0) {
+	return error?.stack ? {
+		message,
+		stack: error.stack
+	} : { message };
+}
+//#endregion
+//#region ../runner/src/runPersistence.ts
+const SHORT_ID_PATTERN = /^r(\d+)$/;
+/**
+* Generate a filesystem-safe, sortable run id combining a UTC timestamp
+* with a short random suffix.
+*/
+function generateRunId() {
+	const now = /* @__PURE__ */ new Date();
+	const pad = (n) => String(n).padStart(2, "0");
+	return `${`${String(now.getUTCFullYear())}-${pad(now.getUTCMonth() + 1)}-${pad(now.getUTCDate())}T${pad(now.getUTCHours())}-${pad(now.getUTCMinutes())}-${pad(now.getUTCSeconds())}Z`}_${Math.random().toString(36).slice(2, 8)}`;
+}
+function parseShortIdNum(shortId) {
+	if (shortId === void 0) return null;
+	const match = SHORT_ID_PATTERN.exec(shortId);
+	if (!match) return null;
+	const num = Number(match[1]);
+	if (!Number.isFinite(num)) return null;
+	return num;
+}
+/**
+* Return the next `shortId` number to assign based on the existing
+* loaded snapshots. Legacy runs that don't match the `r\d+` format are
+* ignored.
+*/
+function nextShortIdFromSnapshots(snapshots) {
+	let maxNum = -1;
+	for (const snapshot of snapshots) {
+		const num = parseShortIdNum(snapshot.manifest.shortId);
+		if (num !== null && num > maxNum) maxNum = num;
+	}
+	return maxNum + 1;
+}
+async function loadPersistedRunSnapshots(localStateDir) {
+	const runsDir = join(localStateDir, "runs");
+	const entriesResult = await resultify(() => readdir(runsDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	const snapshots = [];
+	const runDirs = entriesResult.value.filter((entry) => entry.isDirectory()).map((entry) => join(runsDir, entry.name)).toSorted();
+	for (const runDir of runDirs) {
+		const snapshot = await loadPersistedRunSnapshot(runDir);
+		if (!snapshot) continue;
+		snapshots.push(snapshot);
+	}
+	return snapshots;
+}
+async function persistCaseDetail(runDir, caseDetail) {
+	await writeFile(join(runDir, "case-details", `${encodeCaseDetailFileName(caseDetail.caseId)}.json`), JSON.stringify(caseDetail, null, 2));
+}
+function getLastRunStatuses(params) {
+	const latestRunInfos = getLatestRunInfos(params);
+	return new Map([...latestRunInfos].map(([evalId, info]) => [evalId, info.status]));
+}
+/**
+* Return the latest scoped run metadata for each eval based on persisted and
+* in-memory runs.
+*/
+function getLatestRunInfos(params) {
+	const { runs, knownEvals } = params;
+	const knownEvalMetas = [...knownEvals];
+	const manualScoreKeysByEval = new Map(knownEvalMetas.map((evalMeta) => [evalMeta.id, evalMeta.columnDefs.filter((columnDef) => columnDef.isManualScore === true).map((columnDef) => columnDef.key)]));
+	const orderedRuns = [...runs].toSorted((a, b) => new Date(getRunFreshnessTimestamp(a.manifest)).getTime() - new Date(getRunFreshnessTimestamp(b.manifest)).getTime());
+	const latestRunInfos = /* @__PURE__ */ new Map();
+	for (const run of orderedRuns) for (const evalId of getRunEvalIds(run, knownEvalMetas.map((evalMeta) => evalMeta.id))) latestRunInfos.set(evalId, {
+		status: getEvalStatusForRun(run, evalId, manualScoreKeysByEval.get(evalId) ?? []),
+		startedAt: getRunFreshnessTimestamp(run.manifest),
+		commitSha: run.manifest.commitSha ?? null,
+		evalSourceFingerprint: run.manifest.evalSourceFingerprints[evalId] ?? null
+	});
+	return latestRunInfos;
+}
+function toLastRunStatus$1(status) {
+	return status === "pending" ? null : status;
+}
+async function loadPersistedRunSnapshot(runDir) {
+	const manifest = await readParsedJsonFile(join(runDir, "run.json"), { safeParse: runManifestSchema.safeParse.bind(runManifestSchema) });
+	if (!manifest) return null;
+	const summary = await readParsedJsonFile(join(runDir, "summary.json"), { safeParse: runSummarySchema.safeParse.bind(runSummarySchema) });
+	if (!summary) return null;
+	return {
+		runDir,
+		manifest,
+		summary,
+		cases: await readCaseRows(runDir),
+		caseDetails: await readCaseDetails(runDir)
+	};
+}
+async function readParsedJsonFile(filePath, schema) {
+	const fileResult = await resultify(() => readFile(filePath, "utf-8"));
+	if (fileResult.error) return null;
+	const jsonResult = resultify(() => JSON.parse(fileResult.value));
+	if (jsonResult.error) return null;
+	const parsed = schema.safeParse(jsonResult.value);
+	if (!parsed.success) return null;
+	return parsed.data;
+}
+async function readCaseRows(runDir) {
+	const fileResult = await resultify(() => readFile(join(runDir, "cases.jsonl"), "utf-8"));
+	if (fileResult.error) return [];
+	const rows = [];
+	for (const rawLine of fileResult.value.split("\n")) {
+		const line = rawLine.trim();
+		if (line.length === 0) continue;
+		const jsonResult = resultify(() => JSON.parse(line));
+		if (jsonResult.error) continue;
+		const parsed = caseRowSchema.safeParse(jsonResult.value);
+		if (!parsed.success) continue;
+		rows.push(parsed.data);
+	}
+	return rows;
+}
+async function readCaseDetails(runDir) {
+	const detailsDir = join(runDir, "case-details");
+	const entriesResult = await resultify(() => readdir(detailsDir, { withFileTypes: true }));
+	if (entriesResult.error) return /* @__PURE__ */ new Map();
+	const caseDetails = /* @__PURE__ */ new Map();
+	for (const entry of entriesResult.value) {
+		if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
+		const detail = await readParsedJsonFile(join(detailsDir, entry.name), { safeParse: caseDetailSchema.safeParse.bind(caseDetailSchema) });
+		if (!detail) continue;
+		caseDetails.set(detail.caseId, detail);
+	}
+	return caseDetails;
+}
+function getRunEvalIds(run, knownEvalIds) {
+	const evalIds = new Set(run.cases.map((caseRow) => caseRow.evalId));
+	if (run.manifest.target.mode === "evalIds") for (const evalId of run.manifest.target.evalIds ?? []) evalIds.add(evalId);
+	else if (run.manifest.target.mode === "all" && evalIds.size === 0) for (const evalId of knownEvalIds) evalIds.add(evalId);
+	return [...evalIds];
+}
+function getEvalStatusForRun(run, evalId, manualScoreKeys) {
+	const evalCases = run.cases.filter((caseRow) => caseRow.evalId === evalId);
+	if (evalCases.length > 0) {
+		if (hasPendingManualScores(evalCases, manualScoreKeys)) return "unscored";
+		return toLastRunStatus$1(deriveStatusFromCaseRows({ caseRows: evalCases }));
+	}
+	return toLastRunStatus$1(deriveStatusFromChildStatuses({
+		statuses: [],
+		lifecycleStatus: run.manifest.status
+	}));
+}
+function hasPendingManualScores(caseRows, manualScoreKeys) {
+	if (manualScoreKeys.length === 0) return false;
+	return caseRows.some((caseRow) => manualScoreKeys.some((key) => {
+		const value = caseRow.columns[key];
+		return typeof value !== "number" || !Number.isFinite(value);
+	}));
+}
+function encodeCaseDetailFileName(caseId) {
+	return encodeURIComponent(caseId);
+}
+//#endregion
+//#region ../runner/src/runQueue.ts
+async function executeQueuedCases(params) {
+	const { runState, queuedCases, concurrency, globalTraceDisplay } = params;
+	let nextCaseIndex = 0;
+	let workerError = void 0;
+	const workerCount = Math.min(concurrency, queuedCases.length);
+	const workers = Array.from({ length: workerCount }, async () => {
+		while (!runState.abortController.signal.aborted && workerError === void 0) {
+			const queuedCase = queuedCases[nextCaseIndex];
+			nextCaseIndex += 1;
+			if (queuedCase === void 0) return;
+			try {
+				await executeQueuedCase({
+					queuedCase,
+					runState,
+					globalTraceDisplay
+				});
+			} catch (error) {
+				workerError = error instanceof Error ? error : new Error(String(error));
+				return;
+			}
+		}
+	});
+	await Promise.all(workers);
+	if (workerError instanceof Error) throw workerError;
+	if (workerError !== void 0) throw new Error(typeof workerError === "string" ? workerError : typeof workerError === "number" || typeof workerError === "boolean" || typeof workerError === "bigint" ? String(workerError) : workerError === null ? "null" : "Unknown queue worker error");
+}
+async function executeQueuedCase(params) {
+	const { queuedCase, runState, globalTraceDisplay } = params;
+	const startTime = Date.now();
+	const result = await queuedCase.execute({
+		globalTraceDisplay,
+		signal: runState.abortController.signal,
+		startTime
+	});
+	await queuedCase.onComplete(result);
+}
+//#endregion
+//#region ../runner/src/runOrchestration.ts
+/**
+* Ranks case statuses from worst to best. Used to order trial attempts so the
+* pessimistic (`lowestScore`) strategy can pick the worst attempt. Any
+* non-terminal status outside `pass`/`fail`/`error` is treated as indistinct
+* from `fail` for comparison purposes.
+*/
+function statusRank(status) {
+	if (status === "pass") return 2;
+	if (status === "error") return 0;
+	return 1;
+}
+/**
+* Returns the minimum numeric value across the declared score columns for a
+* trial, or `-Infinity` when no score has a numeric value. Used as a
+* tiebreaker between trials that share the same status.
+*/
+function minScoreValue(caseRow, scoreKeys) {
+	let min = Number.POSITIVE_INFINITY;
+	for (const key of scoreKeys) {
+		const v = caseRow.columns[key];
+		if (typeof v === "number" && Number.isFinite(v)) {
+			if (v < min) min = v;
+		}
+	}
+	return Number.isFinite(min) ? min : Number.NEGATIVE_INFINITY;
+}
+function compareTrialResults(left, right, scoreKeys) {
+	const statusDiff = statusRank(left.caseRow.status) - statusRank(right.caseRow.status);
+	if (statusDiff !== 0) return statusDiff;
+	const scoreDiff = minScoreValue(left.caseRow, scoreKeys) - minScoreValue(right.caseRow, scoreKeys);
+	if (scoreDiff !== 0) return scoreDiff;
+	return left.caseRow.trial - right.caseRow.trial;
+}
+function pickWinningTrial(params) {
+	const orderedAttempts = [...params.attempts].toSorted((left, right) => compareTrialResults(left, right, params.scoreKeys));
+	if (params.strategy === "lowestScore") {
+		const [lowestAttempt] = orderedAttempts;
+		if (lowestAttempt === void 0) throw new Error("Expected at least one trial attempt");
+		return lowestAttempt;
+	}
+	const medianAttempt = orderedAttempts[Math.floor((orderedAttempts.length - 1) / 2)];
+	if (medianAttempt === void 0) throw new Error("Expected at least one trial attempt");
+	return medianAttempt;
+}
+async function executeRun({ runState, request, runDir, config, evals, cacheStore, lastRunStatusMap, latestRunInfoMap, emitEvent, emitDiscoveryEvent, getSourceFingerprint, getConfiguredConcurrency, getSortedEvalMetas, getTargetEvals }) {
+	try {
+		const targetEvals = getTargetEvals(request);
+		emitEvent(runState, {
+			type: "run.started",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: runState.manifest
+		});
+		const allCaseRows = [];
+		const evalErrors = [];
+		const queuedCases = [];
+		const preparedEvals = [];
+		const cacheMode = runState.manifest.cacheMode ?? "use";
+		const cacheEnabled = config.cache?.enabled !== false;
+		for (const evalMeta of targetEvals) {
+			if (runState.abortController.signal.aborted) break;
+			const evalFilePath = evalMeta.sourceFilePath;
+			let codeFingerprint = "";
+			try {
+				codeFingerprint = getSourceFingerprint(await readFile(evalFilePath, "utf-8"));
+			} catch {
+				codeFingerprint = "";
+			}
+			if (codeFingerprint.length > 0) runState.manifest.evalSourceFingerprints[evalMeta.id] = codeFingerprint;
+			else delete runState.manifest.evalSourceFingerprints[evalMeta.id];
+			try {
+				const registry = getEvalRegistry();
+				await loadEvalModule(evalFilePath, codeFingerprint);
+				const entry = registry.get(evalMeta.id);
+				if (!entry) {
+					evalErrors.push({
+						evalId: evalMeta.id,
+						message: `Eval "${evalMeta.id}" was not registered after importing ${evalFilePath}`
+					});
+					continue;
+				}
+				await entry.use(async (evalDef) => {
+					const cases = filterEvalCases(resolveRunnableEvalCases({
+						cases: typeof evalDef.cases === "function" ? await evalDef.cases() : evalDef.cases ?? [],
+						evalId: evalMeta.id
+					}), request.target.evalIds, request.target.caseIds, evalMeta.id);
+					runState.summary.totalCases += cases.length;
+					const accumulatedColumns = /* @__PURE__ */ new Map();
+					const evalCaseRows = [];
+					const preparedCases = [];
+					const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
+					const manualScoreKeys = Object.freeze(Object.keys(evalDef.manualScores ?? {}));
+					preparedEvals.push({
+						evalMeta,
+						accumulatedColumns,
+						evalCaseRows,
+						preparedCases,
+						scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
+						mergeColumns: (columns) => {
+							mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
+						}
+					});
+					for (const evalCase of cases) {
+						if (runState.abortController.signal.aborted) break;
+						const trialResults = [];
+						preparedCases.push({
+							caseId: evalCase.id,
+							trialResults
+						});
+						for (let trial = 0; trial < request.trials; trial++) {
+							const bufferedCacheStore = cacheEnabled && cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
+							queuedCases.push({
+								execute: async ({ startTime, signal, globalTraceDisplay }) => {
+									const { caseDetail, caseRowUpdate } = await runCase({
+										evalDef,
+										evalId: evalMeta.id,
+										evalCase,
+										globalTraceDisplay,
+										trial,
+										signal,
+										startTime,
+										cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
+										cacheMode,
+										codeFingerprint,
+										artifactDir: join(runDir, "artifacts"),
+										runId: runState.manifest.id
+									});
+									return {
+										caseDetail,
+										caseRow: {
+											caseId: evalCase.id,
+											evalId: evalMeta.id,
+											status: caseRowUpdate.status ?? "pending",
+											latencyMs: caseRowUpdate.latencyMs ?? null,
+											columns: caseRowUpdate.columns ?? {},
+											trial
+										}
+									};
+								},
+								onComplete: ({ caseDetail, caseRow }) => {
+									trialResults.push({
+										caseDetail,
+										caseRow,
+										bufferedCacheStore
+									});
+								}
+							});
+						}
+					}
+				});
+			} catch (error) {
+				console.error(`Error running eval ${evalMeta.id}:`, error);
+				evalErrors.push({
+					evalId: evalMeta.id,
+					message: error instanceof Error ? error.message : String(error)
+				});
+				lastRunStatusMap.set(evalMeta.id, "error");
+				latestRunInfoMap.set(evalMeta.id, {
+					status: "error",
+					startedAt: runState.manifest.endedAt ?? runState.manifest.startedAt,
+					commitSha: runState.manifest.commitSha ?? null,
+					evalSourceFingerprint: runState.manifest.evalSourceFingerprints[evalMeta.id] ?? null
+				});
+			}
+		}
+		await executeQueuedCases({
+			runState,
+			queuedCases,
+			concurrency: getConfiguredConcurrency(),
+			globalTraceDisplay: config.traceDisplay
+		});
+		for (const preparedEval of preparedEvals) {
+			for (const preparedCase of preparedEval.preparedCases) {
+				if (preparedCase.trialResults.length === 0) continue;
+				const winningTrial = pickWinningTrial({
+					strategy: runState.manifest.trialSelection,
+					attempts: preparedCase.trialResults,
+					scoreKeys: preparedEval.scoreKeys
+				});
+				if (winningTrial.bufferedCacheStore !== null) await winningTrial.bufferedCacheStore.commit();
+				runState.cases.push(winningTrial.caseRow);
+				runState.caseDetails.set(preparedCase.caseId, winningTrial.caseDetail);
+				preparedEval.mergeColumns(winningTrial.caseDetail.columns);
+				if (winningTrial.caseRow.status === "pass") runState.summary.passedCases++;
+				else if (winningTrial.caseRow.status === "error") runState.summary.errorCases++;
+				else runState.summary.failedCases++;
+				await writeFile(join(runDir, "traces", `${preparedCase.caseId}.json`), JSON.stringify(winningTrial.caseDetail.trace, null, 2));
+				await persistCaseDetail(runDir, winningTrial.caseDetail);
+				emitEvent(runState, {
+					type: "case.finished",
+					runId: runState.manifest.id,
+					timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+					payload: winningTrial.caseRow
+				});
+				preparedEval.evalCaseRows.push(winningTrial.caseRow);
+				allCaseRows.push(winningTrial.caseRow);
+			}
+			preparedEval.evalMeta.columnDefs = [...preparedEval.accumulatedColumns.values()];
+			lastRunStatusMap.set(preparedEval.evalMeta.id, toLastRunStatus(deriveStatusFromCaseRows({ caseRows: preparedEval.evalCaseRows })));
+			const latestStatus = lastRunStatusMap.get(preparedEval.evalMeta.id) ?? null;
+			latestRunInfoMap.set(preparedEval.evalMeta.id, {
+				status: latestStatus,
+				startedAt: runState.manifest.endedAt ?? runState.manifest.startedAt,
+				commitSha: runState.manifest.commitSha ?? null,
+				evalSourceFingerprint: runState.manifest.evalSourceFingerprints[preparedEval.evalMeta.id] ?? null
+			});
+		}
+		const endTime = /* @__PURE__ */ new Date();
+		runState.summary.totalDurationMs = endTime.getTime() - new Date(runState.manifest.startedAt).getTime();
+		const finalStatus = runState.abortController.signal.aborted ? "cancelled" : evalErrors.length > 0 ? "error" : "completed";
+		runState.summary.status = finalStatus;
+		runState.manifest.status = finalStatus;
+		const completedRunAt = endTime.toISOString();
+		runState.manifest.endedAt = completedRunAt;
+		runState.summary.errorMessage = evalErrors.length > 0 ? evalErrors.map((entry) => `[${entry.evalId}] ${entry.message}`).join("\n") : null;
+		for (const evalId of getTargetEvalIds({
+			request,
+			sortedEvalIds: getSortedEvalMetas().map((meta) => meta.id),
+			knownEvalIds: new Set(evals.keys())
+		})) {
+			const latestStatus = lastRunStatusMap.get(evalId) ?? toLastRunStatus(deriveStatusFromCaseRows({
+				caseRows: [],
+				lifecycleStatus: runState.manifest.status
+			}));
+			latestRunInfoMap.set(evalId, {
+				status: latestStatus,
+				startedAt: completedRunAt,
+				commitSha: runState.manifest.commitSha ?? null,
+				evalSourceFingerprint: runState.manifest.evalSourceFingerprints[evalId] ?? null
+			});
+		}
+		emitEvent(runState, {
+			type: "run.summary",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: runState.summary
+		});
+		if (finalStatus === "error") emitEvent(runState, {
+			type: "run.error",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: { message: evalErrors.map((entry) => `[${entry.evalId}] ${entry.message}`).join("\n") }
+		});
+		else emitEvent(runState, {
+			type: "run.finished",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: runState.summary
+		});
+		await persistRunState(runState);
+		emitDiscoveryEvent();
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		runState.manifest.status = "error";
+		runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+		runState.summary.status = "error";
+		runState.summary.errorMessage = message;
+		emitEvent(runState, {
+			type: "run.error",
+			runId: runState.manifest.id,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: { message }
+		});
+		await persistRunState(runState);
+		emitDiscoveryEvent();
+	}
+}
+function toLastRunStatus(status) {
+	return status === "pending" ? null : status;
+}
+//#endregion
+//#region ../runner/src/runner.ts
+/** Create an in-memory eval runner bound to the current workspace config. */
+function createRunner({ watchForChanges = true } = {}) {
+	let config;
+	let workspaceRoot;
+	let localStateDir;
+	let cacheStore;
+	const evals = /* @__PURE__ */ new Map();
+	const runs = /* @__PURE__ */ new Map();
+	const lastRunStatusMap = /* @__PURE__ */ new Map();
+	const latestRunInfoMap = /* @__PURE__ */ new Map();
+	const discoveryListeners = /* @__PURE__ */ new Set();
+	let nextShortIdNum = 0;
+	function toWorkspaceRelativePath(filePath) {
+		return relative(workspaceRoot, filePath).replaceAll("\\", "/");
+	}
+	function getSortedEvalMetas() {
+		return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+	}
+	function getSourceFingerprint(source) {
+		return createHash("sha256").update(source).digest("hex");
+	}
+	function getConfiguredConcurrency() {
+		const configuredConcurrency = config.concurrency;
+		if (typeof configuredConcurrency !== "number" || !Number.isFinite(configuredConcurrency)) return 1;
+		return Math.max(1, Math.floor(configuredConcurrency));
+	}
+	const runner = {
+		async init() {
+			config = await loadConfig();
+			workspaceRoot = config.workspaceRoot ?? process.cwd();
+			localStateDir = resolve(workspaceRoot, ".agent-evals");
+			await mkdir(localStateDir, { recursive: true });
+			await mkdir(join(localStateDir, "runs"), { recursive: true });
+			cacheStore = createFsCacheStore({
+				workspaceRoot,
+				dir: config.cache?.dir
+			});
+			await loadPersistedRuns();
+			await runner.refreshDiscovery();
+			if (watchForChanges) setupWatcher();
+		},
+		async listCache() {
+			return cacheStore.list();
+		},
+		async clearCache(filter) {
+			await cacheStore.clear(filter);
+		},
+		async recomputeStatusesForEval(evalId) {
+			const evalMeta = evals.get(evalId);
+			if (!evalMeta) return { updatedRuns: 0 };
+			const registry = getEvalRegistry();
+			await loadEvalModule(evalMeta.sourceFilePath, evalMeta.sourceFingerprint ?? void 0);
+			const entry = registry.get(evalId);
+			if (!entry) return { updatedRuns: 0 };
+			const scoreThresholds = /* @__PURE__ */ new Map();
+			entry.use((evalDef) => {
+				for (const [key, def] of Object.entries(evalDef.scores ?? {})) {
+					const threshold = normalizeScoreDef(def).passThreshold;
+					if (threshold !== void 0) scoreThresholds.set(key, threshold);
+				}
+				for (const [key, def] of Object.entries(evalDef.manualScores ?? {})) if (def.passThreshold !== void 0) scoreThresholds.set(key, def.passThreshold);
+			});
+			const updatedRuns = await recomputeEvalStatusesInRuns({
+				runs: runs.values(),
+				evalId,
+				evalExists: evals.has(evalId),
+				scoreThresholds,
+				persistCaseDetail
+			});
+			emitDiscoveryEvent();
+			return { updatedRuns };
+		},
+		async cleanRunsForEval(evalId) {
+			let deletedRuns = 0;
+			for (const [runId, run] of [...runs]) {
+				if (!runTouchesEval({
+					target: run.manifest.target,
+					caseRows: run.cases,
+					evalId,
+					evalExists: evals.has(evalId)
+				})) continue;
+				if (run.manifest.status === "running") continue;
+				runs.delete(runId);
+				await rm(run.runDir, {
+					recursive: true,
+					force: true
+				});
+				deletedRuns += 1;
+			}
+			emitDiscoveryEvent();
+			return { deletedRuns };
+		},
+		async updateManualScore({ runId, caseId, scoreKey, value }) {
+			const run = runs.get(runId);
+			if (!run) return {
+				updated: false,
+				reason: "Run not found"
+			};
+			if (run.manifest.status === "running") return {
+				updated: false,
+				reason: "Run is still running"
+			};
+			const caseRow = run.cases.find((row) => row.caseId === caseId);
+			if (!caseRow) return {
+				updated: false,
+				reason: "Case not found"
+			};
+			const evalMeta = evals.get(caseRow.evalId);
+			if (!evalMeta) return {
+				updated: false,
+				reason: "Eval not found"
+			};
+			if (evalMeta.columnDefs.find((def) => def.key === scoreKey)?.isManualScore !== true) return {
+				updated: false,
+				reason: "Manual score not found"
+			};
+			const caseDetail = run.caseDetails.get(caseId);
+			if (!caseDetail) return {
+				updated: false,
+				reason: "Case detail not found"
+			};
+			caseRow.columns[scoreKey] = value;
+			caseDetail.columns[scoreKey] = value;
+			const scoreThresholds = /* @__PURE__ */ new Map();
+			for (const def of evalMeta.columnDefs) {
+				if (def.isScore !== true || def.passThreshold === void 0) continue;
+				scoreThresholds.set(def.key, def.passThreshold);
+			}
+			const nextStatus = recomputePersistedCaseStatus(caseRow, caseDetail, scoreThresholds);
+			caseRow.status = nextStatus;
+			caseDetail.status = nextStatus;
+			const derivedSummary = deriveScopedSummaryFromCases({ caseRows: run.cases });
+			run.summary.totalCases = derivedSummary.totalCases;
+			run.summary.passedCases = derivedSummary.passedCases;
+			run.summary.failedCases = derivedSummary.failedCases;
+			run.summary.errorCases = derivedSummary.errorCases;
+			run.summary.cancelledCases = derivedSummary.cancelledCases;
+			run.summary.totalDurationMs = derivedSummary.totalDurationMs;
+			await persistCaseDetail(run.runDir, caseDetail);
+			await persistRunState(run);
+			emitDiscoveryEvent();
+			return {
+				updated: true,
+				run: {
+					manifest: run.manifest,
+					summary: run.summary,
+					cases: run.cases
+				},
+				caseDetail
+			};
+		},
+		async deleteRun(runId) {
+			const run = runs.get(runId);
+			if (!run) return { deleted: false };
+			if (run.manifest.status === "running") return { deleted: false };
+			runs.delete(runId);
+			await rm(run.runDir, {
+				recursive: true,
+				force: true
+			});
+			emitDiscoveryEvent();
+			return { deleted: true };
+		},
+		getEvals() {
+			const gitState = readGitWorktreeState(workspaceRoot);
+			const result = [];
+			for (const meta of getSortedEvalMetas()) result.push(buildEvalSummary({
+				meta,
+				config,
+				gitState,
+				latestRun: latestRunInfoMap.get(meta.id),
+				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
+			}));
+			return result;
+		},
+		getEval(id) {
+			const meta = evals.get(id);
+			if (!meta) return void 0;
+			return buildEvalSummary({
+				meta,
+				config,
+				gitState: readGitWorktreeState(workspaceRoot),
+				latestRun: latestRunInfoMap.get(meta.id),
+				lastRunStatus: lastRunStatusMap.get(meta.id) ?? null
+			});
+		},
+		async refreshDiscovery() {
+			const patterns = config.include;
+			const discovered = [];
+			for (const pattern of patterns) {
+				const files = await glob(pattern, {
+					cwd: workspaceRoot,
+					absolute: true
+				});
+				discovered.push(...files);
+			}
+			evals.clear();
+			for (const filePath of discovered) try {
+				const content = await readFile(filePath, "utf-8");
+				const discoveredMetas = parseEvalMetas(filePath, content);
+				const sourceFingerprint = getSourceFingerprint(content);
+				const registry = getEvalRegistry();
+				try {
+					await loadEvalModule(filePath, sourceFingerprint);
+				} catch {}
+				for (const meta of discoveredMetas) {
+					const discoveredEntry = registry.get(meta.id);
+					const title = meta.title;
+					let columnDefs = buildDeclaredColumnDefs(void 0, void 0, void 0);
+					let stats;
+					let charts;
+					discoveredEntry?.use((evalDef) => {
+						columnDefs = buildDeclaredColumnDefs(evalDef.columns, evalDef.scores, evalDef.manualScores);
+						stats = evalDef.stats;
+						const validated = validateCharts({
+							charts: evalDef.charts,
+							columnDefs,
+							evalId: meta.id
+						});
+						for (const warning of validated.warnings) console.warn(warning);
+						charts = validated.charts;
+					});
+					evals.set(meta.id, {
+						id: meta.id,
+						title,
+						filePath: toWorkspaceRelativePath(meta.filePath),
+						sourceFilePath: meta.filePath,
+						sourceFingerprint,
+						columnDefs,
+						caseCount: null,
+						stats,
+						charts
+					});
+				}
+			} catch {}
+			emitDiscoveryEvent();
+		},
+		async startRun(request) {
+			const runId = generateRunId();
+			const shortId = `r${String(nextShortIdNum++)}`;
+			const now = (/* @__PURE__ */ new Date()).toISOString();
+			const cacheMode = request.cache?.mode ?? "use";
+			const runDir = join(localStateDir, "runs", runId);
+			const manifest = {
+				id: runId,
+				shortId,
+				status: "running",
+				startedAt: now,
+				endedAt: null,
+				commitSha: readGitWorktreeState(workspaceRoot).commitSha,
+				evalSourceFingerprints: {},
+				target: request.target,
+				trials: request.trials,
+				trialSelection: config.trialSelection ?? "lowestScore",
+				cacheMode
+			};
+			const summary = {
+				runId,
+				status: "running",
+				totalCases: 0,
+				passedCases: 0,
+				failedCases: 0,
+				errorCases: 0,
+				cancelledCases: 0,
+				totalDurationMs: null,
+				errorMessage: null
+			};
+			const abortController = new AbortController();
+			const runState = {
+				runDir,
+				manifest,
+				summary,
+				cases: [],
+				caseDetails: /* @__PURE__ */ new Map(),
+				listeners: /* @__PURE__ */ new Set(),
+				abortController
+			};
+			runs.set(runId, runState);
+			setLatestRunInfoMap({
+				latestRunInfoMap,
+				evalIds: getTargetEvalIds({
+					request,
+					sortedEvalIds: getSortedEvalMetas().map((meta) => meta.id),
+					knownEvalIds: new Set(evals.keys())
+				}),
+				info: {
+					status: "running",
+					startedAt: now,
+					commitSha: manifest.commitSha ?? null,
+					evalSourceFingerprint: null
+				}
+			});
+			await mkdir(runDir, { recursive: true });
+			await mkdir(join(runDir, "traces"), { recursive: true });
+			await mkdir(join(runDir, "artifacts"), { recursive: true });
+			await mkdir(join(runDir, "case-details"), { recursive: true });
+			await writeFile(join(runDir, "run.json"), JSON.stringify(manifest, null, 2));
+			executeRun({
+				runState,
+				request,
+				runDir,
+				config,
+				evals,
+				cacheStore,
+				lastRunStatusMap,
+				latestRunInfoMap,
+				emitEvent,
+				emitDiscoveryEvent,
+				getSourceFingerprint,
+				getConfiguredConcurrency,
+				getSortedEvalMetas,
+				getTargetEvals
+			});
+			return {
+				manifest,
+				summary,
+				cases: []
+			};
+		},
+		getRuns() {
+			return [...runs.values()].map((r) => r.manifest);
+		},
+		getRun(id) {
+			const run = runs.get(id);
+			if (!run) return void 0;
+			return {
+				manifest: run.manifest,
+				summary: run.summary,
+				cases: run.cases
+			};
+		},
+		cancelRun(id) {
+			const run = runs.get(id);
+			if (!run) return;
+			run.abortController.abort();
+			run.manifest.status = "cancelled";
+			run.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
+			run.summary.status = "cancelled";
+			emitEvent(run, {
+				type: "run.cancelled",
+				runId: id,
+				timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+				payload: run.summary
+			});
+		},
+		getCaseDetail(runId, caseId) {
+			const run = runs.get(runId);
+			if (!run) return void 0;
+			return run.caseDetails.get(caseId);
+		},
+		subscribe(runId, listener) {
+			const run = runs.get(runId);
+			if (!run) return () => {};
+			run.listeners.add(listener);
+			return () => {
+				run.listeners.delete(listener);
+			};
+		},
+		subscribeDiscovery(listener) {
+			discoveryListeners.add(listener);
+			return () => {
+				discoveryListeners.delete(listener);
+			};
+		},
+		getWorkspaceRoot() {
+			return workspaceRoot;
+		},
+		getArtifactPath(artifactId_) {
+			return resolveArtifactPath(join(localStateDir, "runs"), artifactId_);
+		}
+	};
+	function setupWatcher() {
+		const watcher = watch(config.include.map((p) => resolve(workspaceRoot, p)), {
+			ignoreInitial: true,
+			persistent: true
+		});
+		watcher.on("change", () => {
+			runner.refreshDiscovery();
+		});
+		watcher.on("add", () => {
+			runner.refreshDiscovery();
+		});
+		watcher.on("unlink", () => {
+			runner.refreshDiscovery();
+		});
+	}
+	function emitDiscoveryEvent() {
+		const lastRunStatuses = getLastRunStatuses({
+			runs: runs.values(),
+			knownEvals: evals.values()
+		});
+		const latestRunInfos = getLatestRunInfos({
+			runs: runs.values(),
+			knownEvals: evals.values()
+		});
+		lastRunStatusMap.clear();
+		for (const [evalId, status] of lastRunStatuses) lastRunStatusMap.set(evalId, status);
+		latestRunInfoMap.clear();
+		for (const [evalId, info] of latestRunInfos) latestRunInfoMap.set(evalId, info);
+		const event = {
+			type: "discovery.updated",
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			payload: runner.getEvals()
+		};
+		for (const listener of discoveryListeners) listener(event);
+	}
+	function getTargetEvals(request) {
+		if (request.target.evalIds && request.target.evalIds.length > 0) return request.target.evalIds.map((id) => evals.get(id)).filter((e) => e !== void 0);
+		return getSortedEvalMetas();
+	}
+	function emitEvent(runState, event) {
+		for (const listener of runState.listeners) try {
+			listener(event);
+		} catch {}
+	}
+	async function loadPersistedRuns() {
+		runs.clear();
+		const persistedRuns = await loadPersistedRunSnapshots(localStateDir);
+		nextShortIdNum = nextShortIdFromSnapshots(persistedRuns);
+		for (const persistedRun of persistedRuns) runs.set(persistedRun.manifest.id, {
+			...persistedRun,
+			listeners: /* @__PURE__ */ new Set(),
+			abortController: new AbortController()
+		});
+	}
+	return runner;
+}
+//#endregion
+//#region src/cli.ts
+function parseArgs(argv) {
+	const args = {
+		command: "help",
+		subcommand: void 0,
+		evalIds: [],
+		caseIds: [],
+		trials: 1,
+		json: false,
+		port: 4100,
+		cacheMode: "use",
+		clearCache: false,
+		all: false
+	};
+	const command = argv[0];
+	if (command === "app" || command === "list" || command === "run" || command === "cache" || command === "help") args.command = command;
+	let cursor = 1;
+	if (args.command === "cache") {
+		const sub = argv[cursor];
+		if (sub === "list" || sub === "clear") {
+			args.subcommand = sub;
+			cursor++;
+		}
+	}
+	for (let i = cursor; i < argv.length; i++) {
+		const arg = argv[i];
+		const next = argv[i + 1];
+		if (arg === "--eval" && next) {
+			args.evalIds.push(...next.split(","));
+			i++;
+		} else if (arg === "--case" && next) {
+			args.caseIds.push(...next.split(","));
+			i++;
+		} else if (arg === "--trials" && next) {
+			args.trials = Number(next);
+			i++;
+		} else if (arg === "--json") args.json = true;
+		else if (arg === "--port" && next) {
+			args.port = Number(next);
+			i++;
+		} else if (arg === "--cache" && next) {
+			if (next === "use" || next === "bypass" || next === "refresh") args.cacheMode = next;
+			i++;
+		} else if (arg === "--no-cache") args.cacheMode = "bypass";
+		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
+		else if (arg === "--clear-cache") args.clearCache = true;
+		else if (arg === "--all") args.all = true;
+	}
+	return args;
+}
+/**
+* Run the Agent Evals CLI against the current workspace.
+*
+* @param argv Raw command-line arguments excluding the executable name.
+*/
+async function runCli(argv) {
+	const args = parseArgs(argv);
+	switch (args.command) {
+		case "app":
+			await commandApp(args);
+			break;
+		case "list":
+			await commandList(args);
+			break;
+		case "run":
+			await commandRun(args);
+			break;
+		case "cache":
+			await commandCache(args);
+			break;
+		default:
+			printHelp();
+			break;
+	}
+}
+const currentDir = dirname(fileURLToPath(import.meta.url));
+const repoRoot = resolve(currentDir, "../../..");
+const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
+function hasRepoWebWorkspace() {
+	return existsSync(resolve(repoRoot, "apps/web/package.json"));
+}
+async function ensureWebUiIsBuilt() {
+	if (!hasRepoWebWorkspace()) return;
+	console.info("Preparing web UI...");
+	await new Promise((resolvePromise, rejectPromise) => {
+		const child = spawn(pnpmCommand, [
+			"--filter",
+			"@agent-evals/web",
+			"build"
+		], {
+			cwd: repoRoot,
+			stdio: "inherit"
+		});
+		child.once("error", (error) => {
+			rejectPromise(error);
+		});
+		child.once("exit", (code, signal) => {
+			if (signal) {
+				rejectPromise(/* @__PURE__ */ new Error(`Web UI build stopped with signal ${signal}.`));
+				return;
+			}
+			if (code !== 0) {
+				rejectPromise(/* @__PURE__ */ new Error(`Web UI build failed with exit code ${String(code)}.`));
+				return;
+			}
+			resolvePromise();
+		});
+	});
+}
+function isHonoAppModule(mod) {
+	if (typeof mod !== "object" || mod === null || !("app" in mod)) return false;
+	const { app } = mod;
+	return typeof app === "object" && app !== null && "fetch" in app && typeof app.fetch === "function";
+}
+function isServerRunnerModule(mod) {
+	if (typeof mod !== "object" || mod === null || !("initRunner" in mod)) return false;
+	return typeof mod.initRunner === "function";
+}
+async function commandApp(args) {
+	await ensureWebUiIsBuilt();
+	const { serve } = await import("@hono/node-server");
+	const bundledWebDist = resolve(currentDir, "apps/web/dist");
+	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
+	const appModule = await import("./app-CKa9TjXw.mjs");
+	const runnerModule = await import("./runner-Ck4X0H3p.mjs");
+	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
+	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
+	await runnerModule.initRunner();
+	console.info(`Agent Evals app: http://localhost:${String(args.port)}`);
+	serve({
+		fetch: appModule.app.fetch,
+		port: args.port
+	});
+}
+async function commandList(args_) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	const evals = runner.getEvals();
+	if (evals.length === 0) {
+		console.info("No eval files found.");
+		return;
+	}
+	console.info("Discovered evals:\n");
+	for (const ev of evals) {
+		const displayStatus = getEvalDisplayStatus({
+			freshnessStatus: ev.freshnessStatus,
+			stale: ev.stale,
+			outdated: ev.outdated,
+			lastRunStatus: ev.lastRunStatus
+		});
+		const title = getEvalTitle(ev);
+		console.info(`  ${title}`);
+		console.info(`    id: ${ev.id}`);
+		console.info(`    file: ${ev.filePath}`);
+		if (displayStatus !== "pending") console.info(`    status: ${displayStatus}`);
+		if (ev.caseCount !== null) console.info(`    cases: ${String(ev.caseCount)}`);
+		console.info("");
+	}
+}
+async function commandRun(args) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	if (args.clearCache) {
+		await runner.clearCache();
+		if (!args.json) {
+			console.info("Cleared cache before run.");
+			console.info("");
+		}
+	}
+	const target = args.caseIds.length > 0 ? {
+		mode: "caseIds",
+		caseIds: args.caseIds,
+		evalIds: args.evalIds.length > 0 ? args.evalIds : void 0
+	} : args.evalIds.length > 0 ? {
+		mode: "evalIds",
+		evalIds: args.evalIds
+	} : { mode: "all" };
+	const run = await runner.startRun({
+		target,
+		trials: args.trials,
+		cache: { mode: args.cacheMode }
+	});
+	if (!args.json) {
+		console.info(`Run started: ${run.manifest.id}`);
+		console.info(`Trials: ${String(args.trials)}`);
+		if (args.cacheMode !== "use") console.info(`Cache mode: ${args.cacheMode}`);
+		console.info("");
+	}
+	await waitForRunCompletion(runner, run.manifest.id);
+	const finalRun = runner.getRun(run.manifest.id);
+	if (!finalRun) {
+		process.exit(1);
+		return;
+	}
+	const { summary } = finalRun;
+	if (args.json) console.info(JSON.stringify(summary, null, 2));
+	else {
+		console.info("--- Run Summary ---");
+		console.info(`Status: ${summary.status}`);
+		console.info(`Total: ${String(summary.totalCases)}`);
+		console.info(`Passed: ${String(summary.passedCases)}`);
+		console.info(`Failed: ${String(summary.failedCases)}`);
+		console.info(`Errors: ${String(summary.errorCases)}`);
+		if (summary.totalCases > 0) console.info(`Pass Rate: ${String(summary.passedCases)}/${String(summary.totalCases)}`);
+		if (summary.totalDurationMs !== null) console.info(`Duration: ${(summary.totalDurationMs / 1e3).toFixed(1)}s`);
+	}
+	if (summary.failedCases > 0 || summary.errorCases > 0) process.exit(1);
+}
+async function commandCache(args) {
+	const runner = createRunner({ watchForChanges: false });
+	await runner.init();
+	if (args.subcommand === "list" || args.subcommand === void 0) {
+		const entries = await runner.listCache();
+		if (args.json) {
+			console.info(JSON.stringify(entries, null, 2));
+			return;
+		}
+		if (entries.length === 0) {
+			console.info("No cache entries.");
+			return;
+		}
+		console.info(`Cache entries (${String(entries.length)}):\n`);
+		for (const entry of entries) {
+			console.info(`  ${entry.namespace}`);
+			console.info(`    key: ${entry.key}`);
+			console.info(`    span: ${entry.spanName} (${entry.spanKind})`);
+			console.info(`    stored: ${entry.storedAt}`);
+			console.info(`    size: ${String(entry.sizeBytes)} bytes`);
+			console.info("");
+		}
+		return;
+	}
+	if (args.subcommand === "clear") {
+		if (args.evalIds.length > 0) {
+			for (const evalId of args.evalIds) {
+				const entries = await runner.listCache();
+				const prefix = `${evalId}__`;
+				const matching = entries.filter((entry) => entry.namespace.startsWith(prefix));
+				for (const entry of matching) await runner.clearCache({
+					namespace: entry.namespace,
+					key: entry.key
+				});
+			}
+			console.info(`Cleared cache entries for: ${args.evalIds.join(", ")}`);
+			return;
+		}
+		if (args.all) {
+			await runner.clearCache();
+			console.info("Cleared all cache entries.");
+			return;
+		}
+		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
+		process.exit(1);
+		return;
+	}
+	printHelp();
+}
+async function waitForRunCompletion(runner, runId) {
+	return new Promise((resolvePromise) => {
+		const check = () => {
+			const run = runner.getRun(runId);
+			if (!run || run.manifest.status === "completed" || run.manifest.status === "cancelled" || run.manifest.status === "error") {
+				resolvePromise();
+				return;
+			}
+			setTimeout(check, 200);
+		};
+		check();
+	});
+}
+function printHelp() {
+	console.info(`
+agent-evals - LLM/Agent eval runner
+
+Commands:
+  app                        Start server with UI
+  list                       List discovered evals
+  run                        Run evals
+  cache list                 List cached operation entries
+  cache clear --eval <id>    Clear cache entries for one eval
+  cache clear --all          Clear every cached entry
+  help                       Show this help
+
+Options:
+  --eval <id>                Run specific eval(s) (comma-separated)
+  --case <id>                Run specific case(s) (comma-separated)
+  --trials <n>               Number of trials per case
+  --json                     Output results as JSON
+  --port <n>                 Server port (default: 4100)
+  --cache <use|bypass|refresh>  Cache mode for this run (default: use)
+  --no-cache                 Shortcut for --cache bypass
+  --refresh-cache            Shortcut for --cache refresh
+  --clear-cache              Clear the cache before starting the run
+  `);
+}
+//#endregion
+export { jsonCellSchema as $, scoreTraceSchema as A, traceAttributeDisplayFormatSchema as B, caseDetailSchema as C, evalStatItemSchema as D, evalStatAggregateSchema as E, evalChartConfigSchema as F, traceDisplayInputConfigSchema as G, traceAttributeDisplayPlacementSchema as H, evalChartMetricSchema as I, cellValueSchema as J, traceSpanKindSchema as K, evalChartTooltipExtraSchema as L, evalChartAxisSchema as M, evalChartBuiltinMetricSchema as N, evalStatsConfigSchema as O, evalChartColorSchema as P, fileRefSchema as Q, evalChartTypeSchema as R, assertionFailureSchema as S, evalFreshnessStatusSchema as T, traceAttributeDisplaySchema as U, traceAttributeDisplayInputSchema as V, traceDisplayConfigSchema as W, columnFormatSchema as X, columnDefSchema as Y, columnKindSchema as Z, cacheModeSchema as _, getEvalRegistry as _t, sseEnvelopeSchema as a, evalTracer as at, serializedCacheSpanSchema as b, deriveScopedSummaryFromCases as c, evalAssert as ct, runManifestSchema as d, isInEvalScope as dt, numberDisplayOptionsSchema as et, runSummarySchema as f, runInEvalScope as ft, cacheListItemSchema as g, defineEval as gt, cacheEntrySchema as h, repoFile as ht, updateManualScoreRequestSchema as i, evalSpan as it, evalChartAggregateSchema as j, evalSummarySchema as k, deriveStatusFromCaseRows as l, getCurrentScope as lt, trialSelectionModeSchema as m, setScopeCacheContext as mt, createRunner as n, runArtifactRefSchema as nt, getEvalTitle as o, hashCacheKey as ot, agentEvalsConfigSchema as p, setEvalOutput as pt, traceSpanSchema as q, createRunRequestSchema as r, buildTraceTree as rt, getEvalDisplayStatus as s, EvalAssertionError as st, runCli as t, repoFileRefSchema as tt, deriveStatusFromChildStatuses as u, incrementEvalOutput as ut, cacheRecordingOpSchema as v, caseRowSchema as w, spanCacheOptionsSchema as x, cacheRecordingSchema as y, evalChartsConfigSchema as z };