@ls-stack/agent-eval 0.55.1 → 0.56.0

package/dist/{runExecution-Sw38bCaq.mjs → runExecution-6lrtj48K.mjs}

@@ -220,6 +220,19 @@ const traceSpanSchema = z.object({
 });
 //#endregion
 //#region ../shared/src/schemas/cache.ts
+const outputColumnOverrideSchema = z.object({
+	label: z.string().optional(),
+	format: columnFormatSchema.optional(),
+	numberFormat: numberDisplayOptionsSchema.optional(),
+	hideInTable: z.boolean().optional(),
+	hideIfNoValue: z.boolean().optional(),
+	align: z.enum([
+		"left",
+		"center",
+		"right"
+	]).optional(),
+	maxStars: z.number().int().min(2).optional()
+});
 /**
 * Mode that controls how the cache is consulted for a given run.
 *
@@ -275,12 +288,15 @@ const traceCacheRefSchema = z.object({
 z.object({
 	key: z.string(),
 	namespace: z.string(),
-	operationType: cacheOperationTypeSchema,
-	operationName: z.string(),
-	spanName: z.string().optional(),
-	spanKind: traceSpanKindSchema.optional(),
 	storedAt: z.string(),
-	sizeBytes: z.number()
+	lastAccessedAt: z.string()
+});
+z.object({
+	removedCacheFiles: z.number(),
+	removedDebugFiles: z.number(),
+	removedBlobFiles: z.number(),
+	removedIndexRows: z.number(),
+	rewrittenIndexes: z.number()
 });
 /** Zod schema for `SerializedCacheSpan`, defined lazily for recursion. */
 const serializedCacheSpanSchema = z.object({
@@ -308,7 +324,8 @@ const cacheRecordingOpSchema = z.discriminatedUnion("kind", [
 	z.object({
 		kind: z.literal("setOutput"),
 		key: z.string(),
-		value: z.unknown()
+		value: z.unknown(),
+		column: outputColumnOverrideSchema.optional()
 	}),
 	z.object({
 		kind: z.literal("appendOutput"),
@@ -789,6 +806,11 @@ const caseRowSchema = z.object({
 	cacheOperations: z.number().optional(),
 	costUsd: z.number().nullable().optional(),
 	columns: z.record(z.string(), cellValueSchema),
+	/**
+	* Runtime column definitions authored by output helpers for this case.
+	* These complement eval-level `columns` without changing discovery metadata.
+	*/
+	outputColumnDefs: z.array(columnDefSchema).optional(),
 	/** Winning trial index for the persisted case result. */
 	trial: z.number()
 });
@@ -895,6 +917,11 @@ const caseDetailSchema = z.object({
 	*/
 	scoringTraces: z.record(z.string(), scoreTraceSchema).optional(),
 	columns: z.record(z.string(), cellValueSchema),
+	/**
+	* Runtime column definitions authored by output helpers for this case.
+	* These complement eval-level `columns` without changing discovery metadata.
+	*/
+	outputColumnDefs: z.array(columnDefSchema).optional(),
 	assertionFailures: z.array(z.union([assertionFailureSchema, legacyAssertionFailureSchema])),
 	/** Logs captured from manual `evalLog(...)` calls and enabled console calls. */
 	logs: z.array(runLogEntrySchema).default([]),
@@ -1377,6 +1404,7 @@ const agentEvalsConfigSchema = z.object({
 		dir: z.string().optional(),
 		maxEntriesPerNamespace: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
 		maxEntriesByNamespace: z.record(z.string(), z.number()).optional(),
+		pruneIdleDelayMs: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
 		maxEntriesPerEval: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional())
 	}).optional()
 });
@@ -2136,17 +2164,34 @@ function computeTokensPerSecond({ outputTokens, durationMs }) {
 	if (durationMs <= 0) return null;
 	return outputTokens / (durationMs / 1e3);
 }
-function readSteps(attributes, path) {
+function readSteps(attributes, path, childModelSteps) {
 	const raw = getNestedAttribute(attributes, path);
 	if (Array.isArray(raw)) return {
 		stepCount: raw.length,
 		stepDetails: raw
 	};
+	if (childModelSteps.length > 0) return {
+		stepCount: childModelSteps.length,
+		stepDetails: childModelSteps
+	};
 	return {
 		stepCount: null,
 		stepDetails: null
 	};
 }
+function buildModelStepsByParent(spans) {
+	const stepsByParent = /* @__PURE__ */ new Map();
+	for (const span of spans) {
+		if (span.kind !== "model_step" || span.parentId === null) continue;
+		const current = stepsByParent.get(span.parentId);
+		if (current === void 0) {
+			stepsByParent.set(span.parentId, [span]);
+			continue;
+		}
+		current.push(span);
+	}
+	return stepsByParent;
+}
 function collectWarnings$1(span) {
 	const out = [];
 	if (span.warning) out.push(span.warning);
@@ -2178,6 +2223,9 @@ function pickError$1(span) {
 * charged twice. Cache read/write costs still contribute to the total USD cost
 * at their configured rates. The `steps` attribute path may resolve to an array
 * of per-step detail objects, with `stepCount` derived from the array length.
+* When a matching LLM span does not expose that array, direct child spans with
+* `kind: 'model_step'` are used as the step details instead. This preserves
+* Mastra/OpenTelemetry traces where model steps are emitted as child spans.
 * `durationMs` and `tokensPerSecond` are `null` while the span is still
 * running. User-defined `metrics` whose path resolves to
 * `undefined` are dropped, but `null`, `0`, and `false` are preserved as
@@ -2186,6 +2234,7 @@ function pickError$1(span) {
 */
 function extractLlmCalls(spans, config) {
 	const kindSet = new Set(config.kinds);
+	const modelStepsByParent = buildModelStepsByParent(spans);
 	const result = [];
 	for (const span of spans) {
 		if (!kindSet.has(span.kind)) continue;
@@ -2271,7 +2320,7 @@ function extractLlmCalls(spans, config) {
 			cachedInputCostUsd,
 			cacheCreationInputCostUsd,
 			reasoningCostUsd,
-			...readSteps(attrs, config.attributes.steps),
+			...readSteps(attrs, config.attributes.steps, modelStepsByParent.get(span.id) ?? []),
 			finishReason: readString$2(attrs, config.attributes.finishReason),
 			durationMs,
 			input: getNestedAttribute(attrs, config.attributes.input),
@@ -3036,6 +3085,7 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		input: options.input,
 		tags: options.tags ?? [],
 		outputs: {},
+		outputColumnOverrides: {},
 		assertionFailures: [],
 		logs: [],
 		spans: [],
@@ -3085,6 +3135,11 @@ function recordOpIfActive(scope, op) {
 	const top = scope.recordingStack.at(-1);
 	if (top) top.ops.push(op);
 }
+function normalizeEvalOutputOptions(options) {
+	if (options === void 0) return void 0;
+	if (typeof options === "string") return { format: options };
+	return options;
+}
 function toAssertionFailure$1(message, error = void 0) {
 	const name = error?.name;
 	const stack = error?.stack ? stripTerminalControlCodes$1(error.stack) : void 0;
@@ -3099,15 +3154,22 @@ function toAssertionFailure$1(message, error = void 0) {
 *
 * Supported values include scalars, JSON-safe objects/arrays, explicit file
 * refs, and native `Blob`/`File` instances for media or file columns.
+*
+* Pass the optional third argument to persist a display format or full column
+* override with this runtime output, for example `'markdown'` or
+* `{ label: 'Receipt', format: 'image', hideInTable: true }`.
 */
-function setEvalOutput(key, value) {
+function setEvalOutput(key, value, options = void 0) {
 	const scope = getCurrentScope();
 	if (!scope) return;
 	scope.outputs[key] = value;
+	const column = normalizeEvalOutputOptions(options);
+	if (column !== void 0) scope.outputColumnOverrides[key] = column;
 	recordOpIfActive(scope, {
 		kind: "setOutput",
 		key,
-		value
+		value,
+		column
 	});
 }
 /**
@@ -4106,6 +4168,7 @@ function replayRecording(scope, parentSpan, recording, options) {
 function applyRecordingOp(scope, parentSpan, op, options) {
 	if (op.kind === "setOutput") {
 		scope.outputs[op.key] = op.value;
+		if (op.column !== void 0) scope.outputColumnOverrides[op.key] = op.column;
 		return;
 	}
 	if (op.kind === "appendOutput") {
@@ -4926,8 +4989,10 @@ const cacheSerializationMarker = "__aecs";
 const supportedCacheSerializationPrefix = "v1:";
 const externalJsonCacheSerializationMarker = "v1:ExternalJson";
 const externalJsonBlobExtension = ".json.br";
+const externalJsonBlobDirName = "cache-blobs";
 const cacheEntryExtension = ".json.br";
 const debugEntryExtension = ".json";
+const cacheIndexFilePrefix = ".index-";
 async function commitPendingCacheWrites(params) {
 	for (const pendingWrite of params.pendingWrites) await params.backingStore.write(pendingWrite.entry, pendingWrite.debugKey);
 }
@@ -4941,8 +5006,14 @@ async function commitPendingCacheWrites(params) {
 function createFsCacheStore(options) {
 	const cacheDir = resolve(options.workspaceRoot, options.dir ?? ".agent-evals/cache");
 	const debugDir = resolve(options.workspaceRoot, options.debugDir ?? ".agent-evals/cache-debug");
-	const blobDir = resolve(options.workspaceRoot, options.blobDir ?? ".agent-evals/cache-blobs");
-	const externalJsonStore = createExternalJsonBlobStore(blobDir);
+	const blobDir = options.blobDir === void 0 ? join(cacheDir, externalJsonBlobDirName) : resolve(options.workspaceRoot, options.blobDir);
+	const legacyBlobDir = resolve(options.workspaceRoot, ".agent-evals/cache-blobs");
+	const fallbackBlobDirs = options.blobDir === void 0 && legacyBlobDir !== blobDir ? [legacyBlobDir] : [];
+	const blobDirs = [blobDir, ...fallbackBlobDirs];
+	const externalJsonStore = createExternalJsonBlobStore({
+		fallbackDirs: fallbackBlobDirs,
+		primaryDir: blobDir
+	});
 	const defaultMaxEntries = normalizeMaxEntries(options.maxEntriesPerNamespace);
 	return {
 		externalJsonStore,
@@ -4956,11 +5027,22 @@ function createFsCacheStore(options) {
 			return blobDir;
 		},
 		async lookup(namespace, keyHash) {
-			const entry = await readCacheEntry(cacheDir, namespace, keyHash);
-			return entry === null ? null : await materializeExternalJsonCacheEntryOrNull(entry, externalJsonStore);
+			const entry = await readIndexedCacheEntry({
+				cacheDir,
+				key: keyHash,
+				namespace
+			});
+			if (entry === null) return null;
+			const materialized = await materializeExternalJsonCacheEntryOrNull(entry, externalJsonStore);
+			if (materialized !== null) await updateCacheIndexLastAccessedAt(cacheDir, namespace, keyHash);
+			return materialized;
 		},
 		async lookupWithDebug(namespace, keyHash) {
-			const rawEntry = await readCacheEntry(cacheDir, namespace, keyHash);
+			const rawEntry = await readIndexedCacheEntry({
+				cacheDir,
+				key: keyHash,
+				namespace
+			});
 			if (rawEntry === null) return null;
 			const entry = await materializeExternalJsonCacheEntryOrNull(rawEntry, externalJsonStore);
 			if (entry === null) return null;
@@ -4975,8 +5057,17 @@ function createFsCacheStore(options) {
 			};
 		},
 		async write(entry, debugKey) {
-			const maxEntries = maxEntriesForNamespace(entry.namespace, defaultMaxEntries, options.maxEntriesByNamespace);
-			await withCacheFileLock(namespaceLockPath(cacheDir, entry.namespace), () => writeCompressedCacheEntry(cacheDir, entry));
+			await withCacheFileLock(namespaceLockPath(cacheDir, entry.namespace), async () => {
+				await writeCompressedCacheEntry(cacheDir, entry);
+				if (!usesSupportedCacheSerialization(entry.recording)) return;
+				const index = await readNamespaceIndex(cacheDir, entry.namespace);
+				index.entries[entry.key] = {
+					storedAt: entry.storedAt,
+					lastAccessedAt: entry.storedAt,
+					blobRefs: await collectExternalJsonBlobRefs(entry, blobDirs)
+				};
+				await writeNamespaceIndex(cacheDir, index);
+			});
 			if (debugKey !== void 0) {
 				if ((await resultify(() => writeDebugKeyEntry({
 					debugDir,
@@ -4987,36 +5078,11 @@ function createFsCacheStore(options) {
 					key: entry.key
 				}));
 			}
-			await pruneEntriesForNamespace({
-				cacheDir,
-				debugDir,
-				namespace: entry.namespace,
-				maxEntries,
-				protectedKey: entry.key
-			});
-			await pruneExternalJsonBlobs(cacheDir, blobDir);
 		},
 		async list() {
-			const files = await listCacheEntryFiles(cacheDir);
 			const items = [];
-			for (const filePath of files) {
-				const fileEntry = await readCacheEntryFilePath(filePath);
-				if (fileEntry === null || !entryMatchesPath(filePath, fileEntry.entry)) continue;
-				const entry = fileEntry.entry;
-				const operationType = entry.operationType ?? "span";
-				const operationName = entry.operationName ?? entry.spanName ?? entry.namespace;
-				items.push({
-					key: entry.key,
-					namespace: entry.namespace,
-					operationType,
-					operationName,
-					spanName: entry.spanName,
-					spanKind: entry.spanKind,
-					storedAt: entry.storedAt,
-					sizeBytes: fileEntry.sizeBytes
-				});
-			}
-			items.sort((a, b) => a.storedAt < b.storedAt ? 1 : -1);
+			for (const index of await listCacheIndexes(cacheDir)) for (const [key, entry] of Object.entries(index.entries)) items.push(toCacheListItem(index.namespace, key, entry));
+			items.sort((a, b) => a.lastAccessedAt < b.lastAccessedAt ? 1 : -1);
 			return items;
 		},
 		async clear(filter) {
@@ -5029,21 +5095,46 @@ function createFsCacheStore(options) {
 					recursive: true,
 					force: true
 				});
-				await rm(blobDir, {
+				await Promise.all(blobDirs.map((dir) => rm(dir, {
 					recursive: true,
 					force: true
-				});
+				})));
 				return;
 			}
 			if (filter.namespace !== void 0) {
 				await clearCacheEntries(cacheDir, filter);
 				await clearDebugEntries(debugDir, filter);
-				await pruneExternalJsonBlobs(cacheDir, blobDir);
+				await pruneUnreferencedExternalJsonBlobs(cacheDir, blobDirs);
 				return;
 			}
 			await clearCacheEntries(cacheDir, filter);
 			await clearDebugEntries(debugDir, filter);
-			await pruneExternalJsonBlobs(cacheDir, blobDir);
+			await pruneUnreferencedExternalJsonBlobs(cacheDir, blobDirs);
+		},
+		async pruneExternalJsonBlobs() {
+			await pruneUnreferencedExternalJsonBlobs(cacheDir, blobDirs);
+		},
+		async pruneRetention() {
+			for (const index_ of await listCacheIndexes(cacheDir)) {
+				const namespace = index_.namespace;
+				const maxEntries = maxEntriesForNamespace(namespace, defaultMaxEntries, options.maxEntriesByNamespace);
+				const keptKeys = await withCacheFileLock(namespaceLockPath(cacheDir, namespace), async () => {
+					return pruneCacheEntriesForNamespace({
+						cacheDir,
+						index: await readNamespaceIndex(cacheDir, namespace),
+						maxEntries
+					});
+				});
+				await withCacheFileLock(namespaceLockPath(debugDir, namespace), () => pruneDebugEntriesForNamespace(debugDir, namespace, keptKeys));
+			}
+			await pruneUnreferencedExternalJsonBlobs(cacheDir, blobDirs);
+		},
+		async repair() {
+			return repairIndexedCache({
+				blobDirs,
+				cacheDir,
+				debugDir
+			});
 		}
 	};
 }
@@ -5126,11 +5217,31 @@ function entryPath(params) {
 	if (filePath !== namespaceDir && !filePath.startsWith(`${namespaceDir}${sep}`)) throw new Error(`Cache entry key escapes namespace directory: ${params.key}`);
 	return filePath;
 }
-async function readCacheEntry(cacheDir, namespace, key) {
-	return (await readCacheEntryFilePath(cacheEntryPath(cacheDir, namespace, key), {
-		namespace,
-		key
-	}))?.entry ?? null;
+function cacheIndexPath(cacheDir, namespace) {
+	return join(namespaceDirPath(cacheDir, namespace), `${cacheIndexFilePrefix}${hashNamespace(namespace)}${debugEntryExtension}`);
+}
+async function readIndexedCacheEntry(params) {
+	return withCacheFileLock(namespaceLockPath(params.cacheDir, params.namespace), async () => {
+		if ((await readNamespaceIndex(params.cacheDir, params.namespace)).entries[params.key] === void 0) return null;
+		const fileEntry = await readCacheEntryFilePath(cacheEntryPath(params.cacheDir, params.namespace, params.key), {
+			namespace: params.namespace,
+			key: params.key
+		});
+		if (fileEntry === null) return null;
+		return fileEntry.entry;
+	});
+}
+async function updateCacheIndexLastAccessedAt(cacheDir, namespace, key) {
+	await withCacheFileLock(namespaceLockPath(cacheDir, namespace), async () => {
+		const index = await readNamespaceIndex(cacheDir, namespace);
+		const entry = index.entries[key];
+		if (entry === void 0) return;
+		index.entries[key] = {
+			...entry,
+			lastAccessedAt: new Date(getRealDateNowMs()).toISOString()
+		};
+		await writeNamespaceIndex(cacheDir, index);
+	});
 }
 async function readCacheEntryFilePath(filePath, expected) {
 	if (!existsSync(filePath)) return null;
@@ -5145,10 +5256,7 @@ async function readCacheEntryFilePath(filePath, expected) {
 	const entry = parsed.data;
 	if (!usesSupportedCacheSerialization(entry.recording)) return null;
 	if (expected !== void 0 && (entry.namespace !== expected.namespace || entry.key !== expected.key)) return null;
-	return {
-		entry,
-		sizeBytes: compressedResult.value.byteLength
-	};
+	return { entry };
 }
 async function writeCompressedCacheEntry(cacheDir, entry) {
 	const filePath = cacheEntryPath(cacheDir, entry.namespace, entry.key);
@@ -5197,23 +5305,132 @@ async function writeAtomicFile(filePath, contents) {
 	await writeFile(tmpPath, contents);
 	await rename(tmpPath, filePath);
 }
+const emptyCacheIndex = (namespace) => ({
+	version: 1,
+	namespace,
+	entries: {}
+});
+async function readNamespaceIndex(cacheDir, namespace) {
+	const indexPath = cacheIndexPath(cacheDir, namespace);
+	if (!existsSync(indexPath)) return emptyCacheIndex(namespace);
+	const rawResult = await resultify(() => readFile(indexPath, "utf8"));
+	if (rawResult.error) return emptyCacheIndex(namespace);
+	return parseCacheIndexFile(safeJsonParse(rawResult.value), namespace) ?? emptyCacheIndex(namespace);
+}
+async function writeNamespaceIndex(cacheDir, index) {
+	const entries = Object.entries(index.entries);
+	if (entries.length === 0) {
+		await rm(cacheIndexPath(cacheDir, index.namespace), { force: true });
+		await removeDirIfEmpty(namespaceDirPath(cacheDir, index.namespace));
+		return;
+	}
+	const sortedEntries = entries.toSorted(([a], [b]) => a < b ? -1 : 1);
+	const normalizedEntries = Object.fromEntries(sortedEntries.map(([key, entry]) => [key, entry]));
+	await writeAtomicFile(cacheIndexPath(cacheDir, index.namespace), JSON.stringify({
+		...index,
+		entries: normalizedEntries
+	}, null, 2));
+}
+async function listCacheIndexes(cacheDir) {
+	if (!existsSync(cacheDir)) return [];
+	const entriesResult = await resultify(() => readdir(cacheDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	const records = [];
+	for (const entry of entriesResult.value) {
+		if (!entry.isDirectory()) continue;
+		const namespaceDir = join(cacheDir, entry.name);
+		for (const indexFilePath of await listCacheIndexFiles(namespaceDir)) {
+			const rawResult = await resultify(() => readFile(indexFilePath, "utf8"));
+			if (rawResult.error) continue;
+			const parsed = parseCacheIndexFile(safeJsonParse(rawResult.value));
+			if (parsed === null) continue;
+			records.push(parsed);
+		}
+	}
+	return records;
+}
+function hashNamespace(namespace) {
+	return createHash("sha256").update(namespace).digest("hex");
+}
+function parseCacheIndexFile(value, expectedNamespace) {
+	if (!isRecordLike(value)) return null;
+	if (value.version !== 1 || typeof value.namespace !== "string") return null;
+	if (expectedNamespace !== void 0 && value.namespace !== expectedNamespace) return null;
+	if (!isRecordLike(value.entries)) return null;
+	const entries = {};
+	for (const [key, entryValue] of Object.entries(value.entries)) {
+		const entry = parseCacheIndexEntry(entryValue);
+		if (entry === null) return null;
+		entries[key] = entry;
+	}
+	return {
+		version: 1,
+		namespace: value.namespace,
+		entries
+	};
+}
+function parseCacheIndexEntry(value) {
+	if (!isRecordLike(value)) return null;
+	if (typeof value.storedAt !== "string" || typeof value.lastAccessedAt !== "string") return null;
+	if (!Array.isArray(value.blobRefs)) return null;
+	const blobRefs = [];
+	for (const blobRef of value.blobRefs) {
+		if (typeof blobRef !== "string") return null;
+		blobRefs.push(blobRef);
+	}
+	return {
+		storedAt: value.storedAt,
+		lastAccessedAt: value.lastAccessedAt,
+		blobRefs
+	};
+}
+function toCacheListItem(namespace, key, entry) {
+	return {
+		key,
+		namespace,
+		storedAt: entry.storedAt,
+		lastAccessedAt: entry.lastAccessedAt
+	};
+}
+function keyFromEntryFilePath(filePath, extension) {
+	const name = basename(filePath);
+	if (!name.endsWith(extension)) return null;
+	return name.slice(0, -extension.length);
+}
+function debugNamespaceFromPath(debugDir, filePath) {
+	return basename(dirname(resolve(debugDir, relative(debugDir, filePath))));
+}
 async function clearCacheEntries(cacheDir, filter) {
-	const files = filter.namespace === void 0 ? await listCacheEntryFiles(cacheDir) : await listCacheEntryFiles(namespaceDirPath(cacheDir, filter.namespace));
-	for (const filePath of files) {
-		const fileEntry = await readCacheEntryFilePath(filePath);
-		if (fileEntry === null) continue;
-		const entry = fileEntry.entry;
-		if (!entryMatchesFilter(entry, filter)) continue;
-		await withCacheFileLock(namespaceLockPath(cacheDir, entry.namespace), () => rm(filePath, { force: true }));
+	const indexes = filter.namespace === void 0 ? await listCacheIndexes(cacheDir) : [await readNamespaceIndex(cacheDir, filter.namespace)];
+	for (const record of indexes) {
+		const namespace = record.namespace;
+		await withCacheFileLock(namespaceLockPath(cacheDir, namespace), async () => {
+			const index = await readNamespaceIndex(cacheDir, namespace);
+			const matchingKeys = Object.keys(index.entries).filter((key) => {
+				return index.entries[key] !== void 0 && entryMatchesFilter({
+					namespace,
+					key
+				}, filter);
+			});
+			for (const key of matchingKeys) {
+				await rm(cacheEntryPath(cacheDir, namespace, key), { force: true });
+				delete index.entries[key];
+			}
+			await writeNamespaceIndex(cacheDir, index);
+		});
 	}
-	if (filter.namespace !== void 0) await removeDirIfEmpty(namespaceDirPath(cacheDir, filter.namespace));
 }
 async function clearDebugEntries(debugDir, filter) {
-	const files = filter.namespace === void 0 ? await listDebugEntryFiles(debugDir) : await listDebugEntryFiles(namespaceDirPath(debugDir, filter.namespace));
+	const files = await listDebugEntryFiles(filter.namespace === void 0 ? debugDir : namespaceDirPath(debugDir, filter.namespace));
 	for (const filePath of files) {
-		const entry = await readDebugEntryFilePath(filePath);
-		if (entry === null || !entryMatchesFilter(entry, filter)) continue;
-		await withCacheFileLock(namespaceLockPath(debugDir, entry.namespace), () => rm(filePath, { force: true }));
+		const namespace = filter.namespace === void 0 ? debugNamespaceFromPath(debugDir, filePath) : filter.namespace;
+		const key = keyFromEntryFilePath(filePath, debugEntryExtension);
+		if (key === null) continue;
+		if (!entryMatchesFilter({
+			namespace,
+			key
+		}, filter)) continue;
+		await withCacheFileLock(namespaceLockPath(debugDir, namespace), () => rm(filePath, { force: true }));
 	}
 	if (filter.namespace !== void 0) await removeDirIfEmpty(namespaceDirPath(debugDir, filter.namespace));
 }
@@ -5221,49 +5438,86 @@ function entryMatchesFilter(entry, filter) {
 	if (filter.namespace !== void 0 && entry.namespace !== filter.namespace) return false;
 	return filter.key === void 0 || entry.key === filter.key;
 }
-async function pruneEntriesForNamespace(params) {
-	const { cacheDir, debugDir, namespace, maxEntries, protectedKey } = params;
-	await withCacheFileLock(namespaceLockPath(cacheDir, namespace), async () => {
-		const keptKeys = await pruneCacheEntriesForNamespace(cacheDir, namespace, maxEntries, protectedKey);
-		await withCacheFileLock(namespaceLockPath(debugDir, namespace), () => pruneDebugEntriesForNamespace(debugDir, namespace, keptKeys));
-	});
-}
-async function pruneCacheEntriesForNamespace(cacheDir, namespace, maxEntries, protectedKey) {
-	const entries = await listCacheEntriesForNamespace(cacheDir, namespace);
-	const sorted = entries.toSorted((a, b) => a.entry.storedAt < b.entry.storedAt ? 1 : -1);
+async function pruneCacheEntriesForNamespace(params) {
+	const { cacheDir, index, maxEntries } = params;
+	const entries = Object.entries(index.entries);
+	const sorted = entries.toSorted(([, a], [, b]) => a.lastAccessedAt < b.lastAccessedAt ? 1 : -1);
 	const keptKeys = /* @__PURE__ */ new Set();
-	const protectedEntry = entries.find((item) => item.entry.key === protectedKey);
-	if (protectedEntry !== void 0) keptKeys.add(protectedEntry.entry.key);
-	for (const item of sorted) {
+	for (const [key] of sorted) {
 		if (keptKeys.size >= maxEntries) break;
-		keptKeys.add(item.entry.key);
+		keptKeys.add(key);
 	}
-	for (const item of entries) if (!keptKeys.has(item.entry.key)) await rm(item.filePath, { force: true });
-	await removeDirIfEmpty(namespaceDirPath(cacheDir, namespace));
+	for (const [key] of entries) if (!keptKeys.has(key)) {
+		await rm(cacheEntryPath(cacheDir, index.namespace, key), { force: true });
+		delete index.entries[key];
+	}
+	await writeNamespaceIndex(cacheDir, index);
 	return keptKeys;
 }
 async function pruneDebugEntriesForNamespace(debugDir, namespace, keptKeys) {
 	const files = await listDebugEntryFiles(namespaceDirPath(debugDir, namespace));
 	for (const filePath of files) {
-		const entry = await readDebugEntryFilePath(filePath);
-		if (entry !== null && entry.namespace === namespace && !keptKeys.has(entry.key)) await rm(filePath, { force: true });
+		const key = keyFromEntryFilePath(filePath, debugEntryExtension);
+		if (key !== null && !keptKeys.has(key)) await rm(filePath, { force: true });
 	}
 	await removeDirIfEmpty(namespaceDirPath(debugDir, namespace));
 }
-async function listCacheEntriesForNamespace(cacheDir, namespace) {
-	const files = await listCacheEntryFiles(namespaceDirPath(cacheDir, namespace));
-	const entries = [];
-	for (const filePath of files) {
-		const fileEntry = await readCacheEntryFilePath(filePath);
-		if (fileEntry !== null && fileEntry.entry.namespace === namespace && entryMatchesPath(filePath, fileEntry.entry)) entries.push({
-			filePath,
-			entry: fileEntry.entry
+async function repairIndexedCache(params) {
+	const summary = {
+		removedCacheFiles: 0,
+		removedDebugFiles: 0,
+		removedBlobFiles: 0,
+		removedIndexRows: 0,
+		rewrittenIndexes: 0
+	};
+	for (const index_ of await listCacheIndexes(params.cacheDir)) {
+		const result = await withCacheFileLock(namespaceLockPath(params.cacheDir, index_.namespace), async () => {
+			const index = await readNamespaceIndex(params.cacheDir, index_.namespace);
+			let removedRows = 0;
+			for (const key of Object.keys(index.entries)) if (!existsSync(cacheEntryPath(params.cacheDir, index.namespace, key))) {
+				delete index.entries[key];
+				removedRows++;
+			}
+			if (removedRows === 0) return {
+				removedRows,
+				rewritten: false
+			};
+			await writeNamespaceIndex(params.cacheDir, index);
+			return {
+				removedRows,
+				rewritten: true
+			};
 		});
+		summary.removedIndexRows += result.removedRows;
+		if (result.rewritten) summary.rewrittenIndexes++;
 	}
-	return entries;
-}
-function entryMatchesPath(filePath, entry) {
-	return basename(filePath) === `${entry.key}${cacheEntryExtension}` && basename(dirname(filePath)) === sanitizeSegment$1(entry.namespace);
+	const indexes = await listCacheIndexes(params.cacheDir);
+	const indexedCacheFiles = /* @__PURE__ */ new Set();
+	const indexedDebugFiles = /* @__PURE__ */ new Set();
+	const indexedBlobRefs = /* @__PURE__ */ new Set();
+	for (const index_ of indexes) for (const [key, entry] of Object.entries(index_.entries)) {
+		indexedCacheFiles.add(cacheEntryPath(params.cacheDir, index_.namespace, key));
+		indexedDebugFiles.add(debugEntryPath(params.debugDir, index_.namespace, key));
+		for (const blobRef of entry.blobRefs) indexedBlobRefs.add(blobRef);
+	}
+	for (const filePath of await listCacheEntryFiles(params.cacheDir, "allNamespaces")) if (!indexedCacheFiles.has(filePath)) {
+		await rm(filePath, { force: true });
+		summary.removedCacheFiles++;
+		await removeDirIfEmpty(dirname(filePath));
+	}
+	for (const filePath of await listDebugEntryFiles(params.debugDir)) if (!indexedDebugFiles.has(filePath)) {
+		await rm(filePath, { force: true });
+		summary.removedDebugFiles++;
+		await removeDirIfEmpty(dirname(filePath));
+	}
+	for (const blobDir of params.blobDirs) {
+		if (!existsSync(blobDir)) continue;
+		for (const blobRef of await listExternalJsonBlobPaths(blobDir)) if (!indexedBlobRefs.has(blobRef)) {
+			await rm(resolveStorePath(blobDir, blobRef), { force: true });
+			summary.removedBlobFiles++;
+		}
+	}
+	return summary;
 }
 function usesSupportedCacheSerialization(value) {
 	if (Array.isArray(value)) return value.every(usesSupportedCacheSerialization);
@@ -5271,14 +5525,14 @@ function usesSupportedCacheSerialization(value) {
 	if (Object.hasOwn(value, cacheSerializationMarker) && (typeof value[cacheSerializationMarker] !== "string" || !value[cacheSerializationMarker].startsWith(supportedCacheSerializationPrefix))) return false;
 	return Object.values(value).every(usesSupportedCacheSerialization);
 }
-function createExternalJsonBlobStore(blobDir) {
+function createExternalJsonBlobStore(params) {
 	return {
 		async write(rawJson) {
 			const rawBytes = Buffer.from(rawJson, "utf8");
 			const hash = hashExternalJson(rawBytes);
 			const path = externalJsonBlobPath(hash);
 			const compressed = brotliCompressSync(rawBytes);
-			const filePath = resolveStorePath(blobDir, path);
+			const filePath = resolveStorePath(params.primaryDir, path);
 			if (!existsSync(filePath)) await writeAtomicFile(filePath, compressed);
 			return {
 				compressedLength: compressed.byteLength,
@@ -5288,10 +5542,15 @@ function createExternalJsonBlobStore(blobDir) {
 			};
 		},
 		async read(ref) {
-			const rawBytes = brotliDecompressSync(await readFile(resolveStorePath(blobDir, ref.path)));
-			const rawJson = rawBytes.toString("utf8");
-			if (rawBytes.byteLength !== ref.length || hashExternalJson(rawBytes) !== ref.hash) throw new Error(`External cache blob failed integrity check: ${ref.hash}`);
-			return rawJson;
+			for (const dir of [params.primaryDir, ...params.fallbackDirs]) {
+				const compressedResult = await resultify(() => readFile(resolveStorePath(dir, ref.path)));
+				if (compressedResult.error) continue;
+				const rawBytesResult = resultify(() => brotliDecompressSync(compressedResult.value));
+				if (rawBytesResult.error) continue;
+				const rawBytes = rawBytesResult.value;
+				if (rawBytes.byteLength === ref.length && hashExternalJson(rawBytes) === ref.hash) return rawBytes.toString("utf8");
+			}
+			throw new Error(`External cache blob failed integrity check: ${ref.hash}`);
 		}
 	};
 }
@@ -5317,28 +5576,55 @@ async function materializeExternalJsonCacheEntryOrNull(entry, store) {
 	const result = await resultify(() => materializeExternalJsonCacheEntry(entry, store));
 	return result.error ? null : result.value;
 }
-async function pruneExternalJsonBlobs(cacheDir, blobDir) {
-	if (!existsSync(blobDir)) return;
+async function pruneUnreferencedExternalJsonBlobs(cacheDir, blobDirs) {
 	const referenced = await collectReferencedExternalJsonBlobPaths(cacheDir);
-	for (const path of await listExternalJsonBlobPaths(blobDir)) if (!referenced.has(path)) await rm(resolveStorePath(blobDir, path), { force: true });
+	for (const blobDir of blobDirs) {
+		if (!existsSync(blobDir)) continue;
+		for (const path of await listExternalJsonBlobPaths(blobDir)) if (!referenced.has(path)) await rm(resolveStorePath(blobDir, path), { force: true });
+	}
 }
 async function collectReferencedExternalJsonBlobPaths(cacheDir) {
 	const paths = /* @__PURE__ */ new Set();
-	for (const filePath of await listCacheEntryFiles(cacheDir)) {
-		const fileEntry = await readCacheEntryFilePath(filePath);
-		if (fileEntry === null || !entryMatchesPath(filePath, fileEntry.entry)) continue;
-		collectExternalJsonBlobPaths(fileEntry.entry, paths);
-	}
+	for (const index_ of await listCacheIndexes(cacheDir)) for (const entry of Object.values(index_.entries)) for (const blobRef of entry.blobRefs) paths.add(blobRef);
 	return paths;
 }
-function collectExternalJsonBlobPaths(value, paths) {
+async function collectExternalJsonBlobRefs(value, blobDirs) {
+	const paths = /* @__PURE__ */ new Set();
+	const pendingBlobPaths = [];
+	collectExternalJsonBlobPaths(value, paths, pendingBlobPaths);
+	while (pendingBlobPaths.length > 0) {
+		const blobPath = pendingBlobPaths.pop();
+		if (blobPath === void 0) continue;
+		const rawJson = await readExternalJsonBlobByPath(blobDirs, blobPath);
+		if (rawJson === null) continue;
+		const json = safeJsonParse(rawJson);
+		if (json === null) continue;
+		collectExternalJsonBlobPaths(json, paths, pendingBlobPaths);
+	}
+	return [...paths].sort();
+}
+function collectExternalJsonBlobPaths(value, paths, pendingBlobPaths) {
 	if (Array.isArray(value)) {
-		for (const item of value) collectExternalJsonBlobPaths(item, paths);
+		for (const item of value) collectExternalJsonBlobPaths(item, paths, pendingBlobPaths);
 		return;
 	}
 	if (!isRecordLike(value)) return;
-	if (value[cacheSerializationMarker] === externalJsonCacheSerializationMarker && typeof value.path === "string") paths.add(value.path);
-	for (const entryValue of Object.values(value)) collectExternalJsonBlobPaths(entryValue, paths);
+	if (value[cacheSerializationMarker] === externalJsonCacheSerializationMarker && typeof value.path === "string") {
+		if (!paths.has(value.path)) {
+			paths.add(value.path);
+			pendingBlobPaths.push(value.path);
+		}
+	}
+	for (const entryValue of Object.values(value)) collectExternalJsonBlobPaths(entryValue, paths, pendingBlobPaths);
+}
+async function readExternalJsonBlobByPath(blobDirs, path) {
+	for (const blobDir of blobDirs) {
+		const compressedResult = await resultify(() => readFile(resolveStorePath(blobDir, path)));
+		if (compressedResult.error) continue;
+		const rawResult = resultify(() => brotliDecompressSync(compressedResult.value).toString("utf8"));
+		if (!rawResult.error) return rawResult.value;
+	}
+	return null;
 }
 async function listExternalJsonBlobPaths(blobDir) {
 	const paths = [];
@@ -5357,12 +5643,33 @@ async function collectExternalJsonBlobFilePaths(root, dir, paths) {
 		if (entry.isFile() && entry.name.endsWith(externalJsonBlobExtension)) paths.push(relative(root, path));
 	}
 }
-async function listCacheEntryFiles(rootDir) {
-	return listFilesWithExtension(rootDir, cacheEntryExtension);
+async function listCacheEntryFiles(rootDir, scope) {
+	if (scope === "namespace") return listDirectFilesWithExtension(rootDir, cacheEntryExtension);
+	if (!existsSync(rootDir)) return [];
+	const entriesResult = await resultify(() => readdir(rootDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	const files = [];
+	for (const entry of entriesResult.value) {
+		if (!entry.isDirectory()) continue;
+		files.push(...await listDirectFilesWithExtension(join(rootDir, entry.name), cacheEntryExtension));
+	}
+	return files;
 }
 async function listDebugEntryFiles(rootDir) {
 	return listFilesWithExtension(rootDir, debugEntryExtension);
 }
+async function listCacheIndexFiles(rootDir) {
+	if (!existsSync(rootDir)) return [];
+	const entriesResult = await resultify(() => readdir(rootDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	return entriesResult.value.filter((entry) => entry.isFile() && entry.name.startsWith(cacheIndexFilePrefix) && entry.name.endsWith(debugEntryExtension)).map((entry) => join(rootDir, entry.name));
+}
+async function listDirectFilesWithExtension(rootDir, extension) {
+	if (!existsSync(rootDir)) return [];
+	const entriesResult = await resultify(() => readdir(rootDir, { withFileTypes: true }));
+	if (entriesResult.error) return [];
+	return entriesResult.value.filter((entry) => entry.isFile() && entry.name.endsWith(extension)).map((entry) => join(rootDir, entry.name));
+}
 async function listFilesWithExtension(rootDir, extension) {
 	if (!existsSync(rootDir)) return [];
 	const entriesResult = await resultify(() => readdir(rootDir, { withFileTypes: true }));
@@ -5396,6 +5703,7 @@ async function withCacheFileLock(filePath, fn) {
 		force: true
 	});
 	if (result.error) throw result.error;
+	return result.value;
 }
 async function acquireLock(lockPath) {
 	const startedAt = Date.now();
@@ -5509,6 +5817,27 @@ function buildDeclaredColumnDefs(overrides, scores, manualScores) {
 	return [...declaredDefs.values()];
 }
 /**
+* Build runtime column definitions from output-level display overrides.
+*
+* These definitions are persisted on case rows/details so `setOutput(...)`
+* can format one-off outputs without adding them to eval discovery metadata.
+*/
+function buildRuntimeOutputColumnDefs(columns, overrides, configuredColumnKeys = /* @__PURE__ */ new Set()) {
+	return Object.entries(overrides).filter(([key]) => columns[key] !== void 0 && !configuredColumnKeys.has(key)).map(([key, override]) => createColumnDef({
+		key,
+		override,
+		inferredKind: inferKindFromFormat(override.format) ?? (override.numberFormat === void 0 ? inferKind(columns[key]) : "number"),
+		isScore: false,
+		isManualScore: false
+	}));
+}
+/** Infer a `ColumnKind` from a runtime value when no override is set. */
+function inferKind(value) {
+	if (typeof value === "number") return "number";
+	if (typeof value === "boolean") return "boolean";
+	return "string";
+}
+/**
 * Coerce an arbitrary runtime value into a serializable `CellValue`.
 * Runtime values use the SDK's tagged serializer so saved run artifacts keep
 * structured data instead of storing JSON strings. Native binary/file root
@@ -6277,7 +6606,7 @@ async function runDeriveFromTracingConfig(params) {
 	}
 }
 async function runCase(params) {
-	const { evalDef, evalId, evalKey = evalId, evalCase, globalTraceDisplay, globalDeriveFromTracing, llmCallsConfig = resolveLlmCallsConfig(void 0), apiCallsConfig = resolveApiCallsConfig(void 0), globalRemoveDefaultConfig, trial, startTime, cacheAdapter, cacheMode, moduleIsolation, evalFilePath, evalFileRelativePath = evalFilePath, workspaceRoot, artifactDir, runId } = params;
+	const { evalDef, evalId, evalKey = evalId, evalCase, globalTraceDisplay, globalColumns, globalDeriveFromTracing, llmCallsConfig = resolveLlmCallsConfig(void 0), apiCallsConfig = resolveApiCallsConfig(void 0), globalRemoveDefaultConfig, trial, startTime, cacheAdapter, cacheMode, moduleIsolation, evalFilePath, evalFileRelativePath = evalFilePath, workspaceRoot, artifactDir, runId } = params;
 	const scopedIdPrefix = buildScopedEvalIdPrefix({
 		evalId,
 		evalFilePath,
@@ -6445,6 +6774,12 @@ async function runCase(params) {
 		if (cell !== void 0) columns[key] = cell;
 	}
 	for (const key of Object.keys(evalDef.manualScores ?? {})) columns[key] = null;
+	const outputColumnDefs = buildRuntimeOutputColumnDefs(columns, scope.outputColumnOverrides, new Set(Object.keys(mergeDefaultColumns({
+		globalColumns,
+		columns: evalDef.columns,
+		globalRemove: globalRemoveDefaultConfig,
+		evalRemove: evalDef.removeDefaultConfig
+	}) ?? {})));
 	const errorInfo = nonAssertError ? {
 		name: nonAssertError.name,
 		message: nonAssertError.message,
@@ -6461,6 +6796,7 @@ async function runCase(params) {
 		trace: displayTrace,
 		traceDisplay,
 		columns,
+		...outputColumnDefs.length > 0 ? { outputColumnDefs } : {},
 		assertionFailures: scope.assertionFailures,
 		logs: scope.logs,
 		error: errorInfo,
@@ -6479,7 +6815,8 @@ async function runCase(params) {
 			durationMs: elapsedMs,
 			cacheHits: cacheHits.length,
 			cacheOperations: cacheEntries.length,
-			columns
+			columns,
+			...outputColumnDefs.length > 0 ? { outputColumnDefs } : {}
 		}
 	};
 }