@ls-stack/agent-eval 0.36.0 → 0.38.0

package/dist/{runOrchestration-V1TxX8es.mjs → runOrchestration-BhUFWvq9.mjs}

@@ -1,14 +1,14 @@
 import { createRequire, registerHooks } from "node:module";
 import { createHash, randomUUID } from "node:crypto";
 import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
-import { extname, isAbsolute, join, relative, resolve } from "node:path";
+import { dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { formatWithOptions, isDeepStrictEqual, stripVTControlCharacters } from "node:util";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { z, z as z$1 } from "zod/v4";
 import { Blob as Blob$1, Buffer as Buffer$1, File as File$1 } from "node:buffer";
-import { gunzipSync, gzipSync } from "node:zlib";
 import { getCompositeKey } from "@ls-stack/utils/getCompositeKey";
 import { existsSync } from "node:fs";
+import { brotliCompressSync, brotliDecompressSync } from "node:zlib";
 import { Result, resultify } from "t-result";
 import { fileURLToPath, pathToFileURL } from "node:url";
 //#region ../sdk/src/defineEval.ts
@@ -867,33 +867,52 @@ function repoFile(path, mimeType) {
 //#endregion
 //#region ../sdk/src/cacheSerialization.ts
 const serializedCacheValueMarker = "__aecs";
-const legacySerializedCacheValueMarker = "__agentEvalsCacheSerialization";
-const jsonSafeCacheValueVersion = "json-safe-v1";
+const jsonSafeCacheValueVersion = "v1";
 const packedNumberArrayMinLength = 128;
-const compressedStringMinBytes = 16 * 1024;
-const compressedJsonMinBytes = 64 * 1024;
-const maxCompressedSizeRatio = .8;
+const maxPackedNumberArraySizeRatio = .8;
+const externalJsonMinChars = 10 * 1024;
+const jsonSafeCacheValueTypes = new Set("ArrayBuffer BigInt Blob Date Error ExternalJson File Float64Array Headers Map Number Object RegExp Set URL URLSearchParams Undefined".split(" "));
 function isRecordLike$3(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function isJsonSafeSerializedCacheValue(value) {
-	return isRecordLike$3(value) && serializationMarkerValue(value) === jsonSafeCacheValueVersion && typeof value.type === "string";
+	return isRecordLike$3(value) && jsonSafeValueType(value) !== void 0;
 }
 function jsonSafeValue(type, value) {
-	return value === void 0 ? {
-		[serializedCacheValueMarker]: jsonSafeCacheValueVersion,
-		type
-	} : {
-		[serializedCacheValueMarker]: jsonSafeCacheValueVersion,
-		type,
+	return value === void 0 ? { [serializedCacheValueMarker]: jsonSafeMarker(type) } : {
+		[serializedCacheValueMarker]: jsonSafeMarker(type),
 		value
 	};
 }
 function hasSerializationMarkerKey(value) {
-	return Object.hasOwn(value, serializedCacheValueMarker) || Object.hasOwn(value, legacySerializedCacheValueMarker);
+	return Object.hasOwn(value, serializedCacheValueMarker);
+}
+function jsonSafeMarker(type) {
+	return `${jsonSafeCacheValueVersion}:${type}`;
+}
+function jsonSafeValueType(value) {
+	const marker = value[serializedCacheValueMarker];
+	if (typeof marker !== "string") return void 0;
+	if (!marker.startsWith(`${jsonSafeCacheValueVersion}:`)) return void 0;
+	const type = marker.slice(3);
+	return isJsonSafeCacheValueType(type) ? type : void 0;
+}
+function isJsonSafeCacheValueType(value) {
+	return jsonSafeCacheValueTypes.has(value);
+}
+function externalJsonRefFromWrapper(value) {
+	const hash = typeof value.hash === "string" ? toExternalJsonHash(value.hash) : void 0;
+	if (hash === void 0 || typeof value.length !== "number" || typeof value.compressedLength !== "number" || typeof value.path !== "string") return;
+	return {
+		compressedLength: value.compressedLength,
+		hash,
+		length: value.length,
+		path: value.path
+	};
 }
-function serializationMarkerValue(value) {
-	return value[serializedCacheValueMarker] ?? value[legacySerializedCacheValueMarker];
+function toExternalJsonHash(value) {
+	if (!value.startsWith("sha256:")) return void 0;
+	return `sha256:${value.slice(7)}`;
 }
 /**
 * Serialize one cached value while keeping plain JSON as plain JSON.
@@ -902,50 +921,57 @@ function serializationMarkerValue(value) {
 * by default; pass `preserveUndefined: true` to round-trip them explicitly.
 */
 async function serializeCacheValue(value, options = void 0) {
-	return serializeJsonSafeValue(value, /* @__PURE__ */ new WeakSet(), 0, normalizeCacheSerializationOptions(options));
-}
-/** Revive one cached value, while preserving legacy JSON-round-tripped data. */
-function deserializeCacheValue(value) {
-	return deserializeJsonSafeValue(value);
-}
-/** Clone one value through the same serialization path used for cache data. */
-async function cloneCacheValue(value, options = void 0) {
-	return deserializeCacheValue(await serializeCacheValue(value, options));
+	return (await serializeJsonSafeValue(value, /* @__PURE__ */ new WeakSet(), 0, normalizeCacheSerializationOptions(options))).value;
 }
-function normalizeCacheSerializationOptions(options) {
+function serializedResult(value, jsonLength = jsonLengthOfSerializedValue(value)) {
 	return {
-		compress: options?.compress !== false,
-		preserveUndefined: options?.preserveUndefined === true
+		value,
+		jsonLength
 	};
 }
+function jsonLengthOfSerializedValue(value) {
+	if (value === void 0) return 0;
+	if (value === null) return 4;
+	if (typeof value === "string") return approximateJsonStringLength(value);
+	return JSON.stringify(value).length;
+}
+function approximateJsonStringLength(value) {
+	return value.length + 2;
+}
+function jsonArrayLength(itemLengths) {
+	return 2 + itemLengths.reduce((total, itemLength) => total + itemLength, 0) + Math.max(itemLengths.length - 1, 0);
+}
+function jsonObjectLength(entries) {
+	return 2 + entries.reduce((total, [key, valueLength]) => total + approximateJsonStringLength(key) + 1 + valueLength, 0) + Math.max(entries.length - 1, 0);
+}
 async function serializeJsonSafeValue(value, refs, depth, config) {
-	if (value === void 0) return config.preserveUndefined ? jsonSafeValue("Undefined") : void 0;
-	if (typeof value === "bigint") return jsonSafeValue("BigInt", value.toString());
-	if (typeof value === "number") return serializeNumber(value);
-	if (typeof value === "string") return serializeString(value, depth, config);
-	if (value instanceof Date) return jsonSafeValue("Date", value.toISOString());
+	if (value === void 0) return config.preserveUndefined ? serializedResult(jsonSafeValue("Undefined")) : serializedResult(void 0);
+	if (typeof value === "bigint") return serializedResult(jsonSafeValue("BigInt", value.toString()));
+	if (typeof value === "number") return serializedResult(serializeNumber(value));
+	if (typeof value === "string") return await externalizeNestedJsonValue(serializedResult(value, approximateJsonStringLength(value)), depth, config);
+	if (value instanceof Date) return serializedResult(jsonSafeValue("Date", value.toISOString()));
 	if (value instanceof Map) return serializeMap(value, refs, depth, config);
 	if (value instanceof Set) return serializeSet(value, refs, depth, config);
-	if (value instanceof RegExp) return jsonSafeValue("RegExp", {
+	if (value instanceof RegExp) return serializedResult(jsonSafeValue("RegExp", {
 		flags: value.flags,
 		source: value.source
-	});
-	if (value instanceof URL) return jsonSafeValue("URL", value.toString());
-	if (value instanceof URLSearchParams) return jsonSafeValue("URLSearchParams", value.toString());
-	if (value instanceof Headers) return jsonSafeValue("Headers", [...value.entries()]);
-	if (value instanceof File) return jsonSafeValue("File", {
+	}));
+	if (value instanceof URL) return serializedResult(jsonSafeValue("URL", value.toString()));
+	if (value instanceof URLSearchParams) return serializedResult(jsonSafeValue("URLSearchParams", value.toString()));
+	if (value instanceof Headers) return serializedResult(jsonSafeValue("Headers", [...value.entries()]));
+	if (value instanceof File) return serializedResult(jsonSafeValue("File", {
 		bytes: await blobToBase64(value),
 		lastModified: value.lastModified,
 		name: value.name,
 		type: value.type
-	});
-	if (value instanceof Blob) return jsonSafeValue("Blob", {
+	}));
+	if (value instanceof Blob) return serializedResult(jsonSafeValue("Blob", {
 		bytes: await blobToBase64(value),
 		type: value.type
-	});
-	if (value instanceof ArrayBuffer) return jsonSafeValue("ArrayBuffer", bytesToBase64(new Uint8Array(value)));
+	}));
+	if (value instanceof ArrayBuffer) return serializedResult(jsonSafeValue("ArrayBuffer", bytesToBase64(new Uint8Array(value))));
 	if (value instanceof Error) return serializeError(value, refs, depth, config);
-	if (!value || typeof value !== "object") return value;
+	if (!value || typeof value !== "object") return serializedResult(value);
 	if (refs.has(value)) throw new Error("Circular cache values are not supported");
 	refs.add(value);
 	if (Array.isArray(value)) {
@@ -953,25 +979,59 @@ async function serializeJsonSafeValue(value, refs, depth, config) {
 			const packed = packNumberArray(value);
 			if (packed !== void 0) {
 				refs.delete(value);
-				return packed;
+				return serializedResult(packed);
 			}
 		}
 		const items = [];
+		const itemLengths = [];
 		for (const item of value) {
 			const serializedItem = await serializeJsonSafeValue(item, refs, depth + 1, config);
-			if (serializedItem !== void 0) items.push(serializedItem);
+			if (serializedItem.value !== void 0) {
+				items.push(serializedItem.value);
+				itemLengths.push(serializedItem.jsonLength);
+			}
 		}
 		refs.delete(value);
-		return compressNestedJsonValue(items, depth, config) ?? items;
+		return await externalizeNestedJsonValue(serializedResult(items, jsonArrayLength(itemLengths)), depth, config);
 	}
 	const entries = [];
+	const entryLengths = [];
 	for (const [key, entryValue] of Object.entries(value)) {
 		const serializedEntryValue = await serializeJsonSafeValue(entryValue, refs, depth + 1, config);
-		if (serializedEntryValue !== void 0) entries.push([key, serializedEntryValue]);
+		if (serializedEntryValue.value !== void 0) {
+			entries.push([key, serializedEntryValue.value]);
+			entryLengths.push([key, serializedEntryValue.jsonLength]);
+		}
 	}
 	refs.delete(value);
-	const serialized = hasSerializationMarkerKey(value) ? jsonSafeValue("Object", entries) : Object.fromEntries(entries);
-	return compressNestedJsonValue(serialized, depth, config) ?? serialized;
+	if (hasSerializationMarkerKey(value)) return await externalizeNestedJsonValue(serializedResult(jsonSafeValue("Object", entries)), depth, config);
+	return await externalizeNestedJsonValue(serializedResult(Object.fromEntries(entries), jsonObjectLength(entryLengths)), depth, config);
+}
+/** Revive one cached value, while preserving legacy JSON-round-tripped data. */
+function deserializeCacheValue(value) {
+	return deserializeJsonSafeValue(value);
+}
+/** Replace external JSON blob refs with their parsed serialized payloads. */
+async function materializeExternalJsonValues(value, store) {
+	if (isJsonSafeSerializedCacheValue(value) && jsonSafeValueType(value) === "ExternalJson") {
+		const ref = externalJsonRefFromWrapper(value);
+		if (ref === void 0) return value;
+		return materializeExternalJsonValues(JSON.parse(await store.read(ref)), store);
+	}
+	if (Array.isArray(value)) return Promise.all(value.map((item) => materializeExternalJsonValues(item, store)));
+	if (!isRecordLike$3(value)) return value;
+	return Object.fromEntries(await Promise.all(Object.entries(value).map(async ([key, entryValue]) => [key, await materializeExternalJsonValues(entryValue, store)])));
+}
+/** Clone one value through the same serialization path used for cache data. */
+async function cloneCacheValue(value, options = void 0) {
+	return deserializeCacheValue(await serializeCacheValue(value, options));
+}
+function normalizeCacheSerializationOptions(options) {
+	return {
+		compress: options?.compress !== false,
+		externalJsonStore: options?.externalJsonStore,
+		preserveUndefined: options?.preserveUndefined === true
+	};
 }
 function serializeNumber(value) {
 	if (Number.isNaN(value)) return jsonSafeValue("Number", "NaN");
@@ -980,11 +1040,6 @@ function serializeNumber(value) {
 	if (Object.is(value, -0)) return jsonSafeValue("Number", "-0");
 	return value;
 }
-function serializeString(value, depth, config) {
-	if (depth === 0) return value;
-	if (!config.compress) return value;
-	return compressNestedStringValue(value) ?? value;
-}
 function isDenseNumberArray(value) {
 	for (let index = 0; index < value.length; index++) if (typeof value[index] !== "number") return false;
 	return true;
@@ -997,48 +1052,32 @@ function encodeFloat64Array(value) {
 }
 function packNumberArray(value) {
 	const serialized = {
-		[serializedCacheValueMarker]: jsonSafeCacheValueVersion,
+		[serializedCacheValueMarker]: jsonSafeMarker("Float64Array"),
 		length: value.length,
-		type: "Float64Array",
 		value: encodeFloat64Array(value)
 	};
-	return compressionIsWorthIt(serialized, Buffer$1.byteLength(JSON.stringify(value))) ? serialized : void 0;
+	return JSON.stringify(serialized).length < JSON.stringify(value).length * maxPackedNumberArraySizeRatio ? serialized : void 0;
 }
 function decodeFloat64Array(value, length) {
 	const bytes = base64ToArrayBuffer(value);
 	const view = new DataView(bytes);
 	return Array.from({ length }, (_, index) => view.getFloat64(index * 8, true));
 }
-function compressNestedStringValue(value) {
-	const rawSize = Buffer$1.byteLength(JSON.stringify(value));
-	if (rawSize < compressedStringMinBytes) return void 0;
-	const compressed = gzipSync(value);
-	const serialized = {
-		[serializedCacheValueMarker]: jsonSafeCacheValueVersion,
-		codec: "gzip",
-		length: Buffer$1.byteLength(value),
-		type: "CompressedString",
-		value: compressed.toString("base64")
+async function externalizeNestedJsonValue(result, depth, config) {
+	if (depth === 0 || !config.compress || config.externalJsonStore === void 0 || result.jsonLength < externalJsonMinChars) return result;
+	const raw = JSON.stringify(result.value);
+	if (raw.length < externalJsonMinChars) return {
+		...result,
+		jsonLength: raw.length
 	};
-	return compressionIsWorthIt(serialized, rawSize) ? serialized : void 0;
-}
-function compressNestedJsonValue(value, depth, config) {
-	if (depth === 0) return void 0;
-	if (!config.compress) return void 0;
-	const raw = JSON.stringify(value);
-	const rawSize = Buffer$1.byteLength(raw);
-	if (rawSize < compressedJsonMinBytes) return void 0;
-	const serialized = {
-		[serializedCacheValueMarker]: jsonSafeCacheValueVersion,
-		codec: "gzip",
-		length: rawSize,
-		type: "CompressedJson",
-		value: gzipSync(raw).toString("base64")
-	};
-	return compressionIsWorthIt(serialized, rawSize) ? serialized : void 0;
-}
-function compressionIsWorthIt(value, rawSize) {
-	return Buffer$1.byteLength(JSON.stringify(value)) < rawSize * maxCompressedSizeRatio;
+	const ref = await config.externalJsonStore.write(raw);
+	return serializedResult({
+		[serializedCacheValueMarker]: jsonSafeMarker("ExternalJson"),
+		compressedLength: ref.compressedLength,
+		hash: ref.hash,
+		length: ref.length,
+		path: ref.path
+	});
 }
 async function serializeMap(value, refs, depth, config) {
 	if (refs.has(value)) throw new Error("Circular cache values are not supported");
@@ -1047,10 +1086,10 @@ async function serializeMap(value, refs, depth, config) {
 	for (const [key, entryValue] of value.entries()) {
 		const serializedKey = await serializeJsonSafeValue(key, refs, depth + 1, config);
 		const serializedEntryValue = await serializeJsonSafeValue(entryValue, refs, depth + 1, config);
-		if (serializedKey !== void 0 && serializedEntryValue !== void 0) entries.push([serializedKey, serializedEntryValue]);
+		if (serializedKey.value !== void 0 && serializedEntryValue.value !== void 0) entries.push([serializedKey.value, serializedEntryValue.value]);
 	}
 	refs.delete(value);
-	return jsonSafeValue("Map", entries);
+	return serializedResult(jsonSafeValue("Map", entries));
 }
 async function serializeSet(value, refs, depth, config) {
 	if (refs.has(value)) throw new Error("Circular cache values are not supported");
@@ -1058,10 +1097,10 @@ async function serializeSet(value, refs, depth, config) {
 	const items = [];
 	for (const item of value.values()) {
 		const serializedItem = await serializeJsonSafeValue(item, refs, depth + 1, config);
-		if (serializedItem !== void 0) items.push(serializedItem);
+		if (serializedItem.value !== void 0) items.push(serializedItem.value);
 	}
 	refs.delete(value);
-	return jsonSafeValue("Set", items);
+	return serializedResult(jsonSafeValue("Set", items));
 }
 async function serializeError(value, refs, depth, config) {
 	if (refs.has(value)) throw new Error("Circular cache values are not supported");
@@ -1070,17 +1109,17 @@ async function serializeError(value, refs, depth, config) {
 	for (const [key, entryValue] of Object.entries(value)) {
 		if (key === "cause") continue;
 		const serializedEntryValue = await serializeJsonSafeValue(entryValue, refs, depth + 1, config);
-		if (serializedEntryValue !== void 0) props.push([key, serializedEntryValue]);
+		if (serializedEntryValue.value !== void 0) props.push([key, serializedEntryValue.value]);
 	}
 	const serialized = jsonSafeValue("Error", {
-		cause: "cause" in value ? await serializeJsonSafeValue(value.cause, refs, depth + 1, config) : void 0,
+		cause: "cause" in value ? (await serializeJsonSafeValue(value.cause, refs, depth + 1, config)).value : void 0,
 		message: value.message,
 		name: value.name,
 		props,
 		stack: value.stack
 	});
 	refs.delete(value);
-	return serialized;
+	return serializedResult(serialized);
 }
 async function blobToBase64(value) {
 	return bytesToBase64(new Uint8Array(await value.arrayBuffer()));
@@ -1101,14 +1140,13 @@ function deserializeJsonSafeValue(value) {
 	return Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, deserializeJsonSafeValue(entryValue)]));
 }
 function deserializeJsonSafeWrapper(value) {
-	switch (value.type) {
+	switch (jsonSafeValueType(value)) {
 		case "ArrayBuffer": return deserializeArrayBuffer(value.value);
 		case "BigInt": return typeof value.value === "string" ? BigInt(value.value) : value.value;
 		case "Blob": return deserializeBlob(value.value);
-		case "CompressedJson": return deserializeCompressedJson(value.value);
-		case "CompressedString": return deserializeCompressedString(value.value);
 		case "Date": return typeof value.value === "string" ? new Date(value.value) : value.value;
 		case "Error": return deserializeError(value.value);
+		case "ExternalJson": return value;
 		case "File": return deserializeFile(value.value);
 		case "Float64Array": return deserializeFloat64Array(value.value, value.length);
 		case "Headers": return new Headers(deserializeStringPairArray(value.value));
@@ -1120,6 +1158,7 @@ function deserializeJsonSafeWrapper(value) {
 		case "URL": return typeof value.value === "string" ? new URL(value.value) : value.value;
 		case "URLSearchParams": return typeof value.value === "string" ? new URLSearchParams(value.value) : value.value;
 		case "Undefined": return;
+		default: return value;
 	}
 }
 function deserializeNumber(value) {
@@ -1129,14 +1168,6 @@ function deserializeNumber(value) {
 	if (value === "-0") return -0;
 	return value;
 }
-function deserializeCompressedString(value) {
-	if (typeof value !== "string") return value;
-	return gunzipSync(Buffer$1.from(value, "base64")).toString("utf8");
-}
-function deserializeCompressedJson(value) {
-	if (typeof value !== "string") return value;
-	return deserializeJsonSafeValue(JSON.parse(gunzipSync(Buffer$1.from(value, "base64")).toString("utf8")));
-}
 function deserializeFloat64Array(value, length) {
 	if (typeof value !== "string" || typeof length !== "number") return value;
 	return decodeFloat64Array(value, length);
@@ -1766,7 +1797,7 @@ function createTraceCache(generateSpanId) {
 				operationType: "value",
 				operationName: info.name,
 				storedAt: new Date(getRealDateNowMs()).toISOString(),
-				recording: await serializeCacheRecording(recording)
+				recording: await serializeCacheRecording(recording, { externalJsonStore: cacheCtx.adapter.externalJsonStore })
 			}, {
 				rawKey: info.key,
 				operationType: "value",
@@ -2209,7 +2240,7 @@ async function traceSpanInternal(info, fn) {
 					spanName: info.name,
 					spanKind: info.kind,
 					storedAt: new Date(getRealDateNowMs()).toISOString(),
-					recording: await serializeCacheRecording(recording)
+					recording: await serializeCacheRecording(recording, { externalJsonStore: ctx.adapter.externalJsonStore })
 				};
 				await ctx.adapter.write(entry, {
 					rawKey: cacheOpts.key,
@@ -2980,6 +3011,8 @@ const evalSummarySchema = z.object({
 	currentCommitSha: z.string().nullable(),
 	columnDefs: z.array(columnDefSchema),
 	caseCount: z.number().nullable(),
+	/** Authored case ids discovered for this eval, when case generation has run. */
+	caseIds: z.array(z.string()).optional(),
 	lastRunStatus: z.enum([
 		"pass",
 		"fail",
@@ -3626,6 +3659,11 @@ const runManifestSchema = z.object({
 		"cancelled",
 		"error"
 	]),
+	/**
+	* Temporary runs are persisted like normal runs, but are deleted before the
+	* next run starts. Older persisted runs default to durable history.
+	*/
+	temporary: z.boolean().optional().default(false),
 	startedAt: z.string(),
 	endedAt: z.string().nullable(),
 	/**
@@ -4521,6 +4559,11 @@ const createRunRequestSchema = z.object({
 	}),
 	trials: z.number().min(1),
 	/**
+	* Persist this run as temporary history. Temporary runs are visible while
+	* present, then deleted before the next run of any kind starts.
+	*/
+	temporary: z.boolean().optional(),
+	/**
 	* Optional cache controls for the run. When omitted, the cache is used in
 	* its default read-through / write-on-miss mode.
 	*/
@@ -4539,8 +4582,9 @@ const updateManualScoreRequestSchema = z.object({ value: z.number().min(0).max(1
 //#region ../runner/src/cacheStore.ts
 const defaultMaxEntriesPerNamespace = 100;
 const cacheSerializationMarker = "__aecs";
-const legacyCacheSerializationMarker = "__agentEvalsCacheSerialization";
-const supportedCacheSerializationVersion = "json-safe-v1";
+const supportedCacheSerializationPrefix = "v1:";
+const externalJsonCacheSerializationMarker = "v1:ExternalJson";
+const externalJsonBlobExtension = ".json.br";
 /**
 * Create a filesystem-backed cache adapter rooted at `<workspaceRoot>/<dir>`.
 *
@@ -4551,20 +4595,28 @@ const supportedCacheSerializationVersion = "json-safe-v1";
 function createFsCacheStore(options) {
 	const cacheDir = resolve(options.workspaceRoot, options.dir ?? ".agent-evals/cache");
 	const debugDir = resolve(options.workspaceRoot, options.debugDir ?? ".agent-evals/cache-debug");
+	const blobDir = resolve(options.workspaceRoot, options.blobDir ?? ".agent-evals/cache-blobs");
+	const externalJsonStore = createExternalJsonBlobStore(blobDir);
 	const defaultMaxEntries = normalizeMaxEntries(options.maxEntriesPerNamespace);
 	return {
+		externalJsonStore,
 		dir() {
 			return cacheDir;
 		},
 		debugDir() {
 			return debugDir;
 		},
+		blobDir() {
+			return blobDir;
+		},
 		async lookup(namespace, keyHash) {
-			return (await readCacheFile(cacheDir, ownerFromNamespace(namespace)))?.entries[keyHash] ?? null;
+			const entry = (await readCacheFile(cacheDir, ownerFromNamespace(namespace)))?.entries[keyHash] ?? null;
+			return entry === null ? null : await materializeExternalJsonCacheEntry(entry, externalJsonStore);
 		},
 		async lookupWithDebug(namespace, keyHash) {
 			const owner = ownerFromNamespace(namespace);
-			const entry = (await readCacheFile(cacheDir, owner))?.entries[keyHash] ?? null;
+			const rawEntry = (await readCacheFile(cacheDir, owner))?.entries[keyHash] ?? null;
+			const entry = rawEntry === null ? null : await materializeExternalJsonCacheEntry(rawEntry, externalJsonStore);
 			if (entry === null) return null;
 			const debugKey = (await readDebugKeyFile(debugDir, owner))?.entries[keyHash];
 			const deserializedEntry = {
@@ -4590,6 +4642,7 @@ function createFsCacheStore(options) {
 					}, entry.namespace, maxEntriesForNamespace(entry.namespace, defaultMaxEntries, options.maxEntriesByNamespace), entry.key)
 				});
 			});
+			await pruneExternalJsonBlobs(cacheDir, blobDir);
 			if (debugKey !== void 0) {
 				if ((await resultify(() => writeDebugKeyEntry({
 					debugDir,
@@ -4641,6 +4694,10 @@ function createFsCacheStore(options) {
 					recursive: true,
 					force: true
 				});
+				await rm(blobDir, {
+					recursive: true,
+					force: true
+				});
 				return;
 			}
 			if (filter.namespace !== void 0) {
@@ -4659,6 +4716,7 @@ function createFsCacheStore(options) {
 					});
 				});
 				await clearDebugEntries(debugDir, filter);
+				await pruneExternalJsonBlobs(cacheDir, blobDir);
 				return;
 			}
 			if (existsSync(cacheDir)) {
@@ -4679,6 +4737,7 @@ function createFsCacheStore(options) {
 				}
 			}
 			await clearDebugEntries(debugDir, filter);
+			await pruneExternalJsonBlobs(cacheDir, blobDir);
 		}
 	};
 }
@@ -4692,9 +4751,10 @@ function createFsCacheStore(options) {
 function createBufferedCacheStore(backingStore) {
 	const pendingEntries = /* @__PURE__ */ new Map();
 	return {
+		externalJsonStore: backingStore.externalJsonStore,
 		async lookup(namespace, keyHash) {
 			const buffered = pendingEntries.get(toPendingKey(namespace, keyHash));
-			if (buffered !== void 0) return buffered.entry;
+			if (buffered !== void 0) return backingStore.externalJsonStore === void 0 ? buffered.entry : await materializeExternalJsonCacheEntry(buffered.entry, backingStore.externalJsonStore);
 			return backingStore.lookup(namespace, keyHash);
 		},
 		write(entry, debugKey) {
@@ -4733,6 +4793,95 @@ function toPendingKey(namespace, keyHash) {
 function sanitizeSegment$1(segment) {
 	return segment.replace(/[^a-zA-Z0-9_.-]/g, "_");
 }
+function createExternalJsonBlobStore(blobDir) {
+	return {
+		async write(rawJson) {
+			const rawBytes = Buffer.from(rawJson, "utf8");
+			const hash = hashExternalJson(rawBytes);
+			const path = externalJsonBlobPath(hash);
+			const compressed = brotliCompressSync(rawBytes);
+			const filePath = resolveStorePath(blobDir, path);
+			if (!existsSync(filePath)) {
+				await mkdir(dirname(filePath), { recursive: true });
+				const tmpPath = `${filePath}.${process.pid.toString()}.tmp`;
+				await writeFile(tmpPath, compressed);
+				await rename(tmpPath, filePath);
+			}
+			return {
+				compressedLength: compressed.byteLength,
+				hash,
+				length: rawBytes.byteLength,
+				path
+			};
+		},
+		async read(ref) {
+			const rawBytes = brotliDecompressSync(await readFile(resolveStorePath(blobDir, ref.path)));
+			const rawJson = rawBytes.toString("utf8");
+			if (rawBytes.byteLength !== ref.length || hashExternalJson(rawBytes) !== ref.hash) throw new Error(`External cache blob failed integrity check: ${ref.hash}`);
+			return rawJson;
+		}
+	};
+}
+function hashExternalJson(rawBytes) {
+	return `sha256:${createHash("sha256").update(rawBytes).digest("hex")}`;
+}
+function externalJsonBlobPath(hash) {
+	const digest = hash.slice(7);
+	return join("sha256", digest.slice(0, 2), `${digest}${externalJsonBlobExtension}`);
+}
+function resolveStorePath(root, relativePath) {
+	const path = resolve(root, relativePath);
+	if (path !== root && !path.startsWith(`${root}${sep}`)) throw new Error(`External cache blob path escapes store: ${relativePath}`);
+	return path;
+}
+async function materializeExternalJsonCacheEntry(entry, store) {
+	return {
+		...entry,
+		recording: cacheRecordingSchema.parse(await materializeExternalJsonValues(entry.recording, store))
+	};
+}
+async function pruneExternalJsonBlobs(cacheDir, blobDir) {
+	if (!existsSync(blobDir)) return;
+	const referenced = await collectReferencedExternalJsonBlobPaths(cacheDir);
+	for (const path of await listExternalJsonBlobPaths(blobDir)) if (!referenced.has(path)) await rm(resolveStorePath(blobDir, path), { force: true });
+}
+async function collectReferencedExternalJsonBlobPaths(cacheDir) {
+	const paths = /* @__PURE__ */ new Set();
+	if (!existsSync(cacheDir)) return paths;
+	const files = await readdir(cacheDir);
+	for (const fileName of files) {
+		if (!fileName.endsWith(".json")) continue;
+		const cacheFile = await readCacheFilePath(join(cacheDir, fileName));
+		if (cacheFile === null) continue;
+		collectExternalJsonBlobPaths(cacheFile, paths);
+	}
+	return paths;
+}
+function collectExternalJsonBlobPaths(value, paths) {
+	if (Array.isArray(value)) {
+		for (const item of value) collectExternalJsonBlobPaths(item, paths);
+		return;
+	}
+	if (!isRecordLike(value)) return;
+	if (value[cacheSerializationMarker] === externalJsonCacheSerializationMarker && typeof value.path === "string") paths.add(value.path);
+	for (const entryValue of Object.values(value)) collectExternalJsonBlobPaths(entryValue, paths);
+}
+async function listExternalJsonBlobPaths(blobDir) {
+	const paths = [];
+	await collectExternalJsonBlobFilePaths(blobDir, blobDir, paths);
+	return paths;
+}
+async function collectExternalJsonBlobFilePaths(root, dir, paths) {
+	const entries = await readdir(dir, { withFileTypes: true });
+	for (const entry of entries) {
+		const path = join(dir, entry.name);
+		if (entry.isDirectory()) {
+			await collectExternalJsonBlobFilePaths(root, path, paths);
+			continue;
+		}
+		if (entry.isFile() && entry.name.endsWith(externalJsonBlobExtension)) paths.push(relative(root, path));
+	}
+}
 async function readCacheFile(cacheDir, owner) {
 	return readCacheFilePath(ownerPath(cacheDir, owner));
 }
@@ -4752,7 +4901,7 @@ async function readCacheFilePath(filePath) {
 function usesSupportedCacheSerialization(value) {
 	if (Array.isArray(value)) return value.every(usesSupportedCacheSerialization);
 	if (!isRecordLike(value)) return true;
-	for (const marker of [cacheSerializationMarker, legacyCacheSerializationMarker]) if (Object.hasOwn(value, marker) && value[marker] !== supportedCacheSerializationVersion) return false;
+	if (Object.hasOwn(value, cacheSerializationMarker) && (typeof value[cacheSerializationMarker] !== "string" || !value[cacheSerializationMarker].startsWith(supportedCacheSerializationPrefix))) return false;
 	return Object.values(value).every(usesSupportedCacheSerialization);
 }
 async function writeOrRemoveCacheFile(cacheDir, cacheFile) {
@@ -6188,6 +6337,27 @@ function runTouchesEval(params) {
 	if (params.target.mode === "evalIds") return params.target.evalKeys?.includes(params.evalKey) ?? params.target.evalIds?.includes(params.evalId ?? params.evalKey) ?? false;
 	return false;
 }
+async function deleteTemporaryRuns(params) {
+	let deletedRuns = 0;
+	for (const [runId, run] of [...params.runs]) {
+		if (run.manifest.temporary !== true) continue;
+		if (run.manifest.status === "running") {
+			const endedAt = /* @__PURE__ */ new Date();
+			run.manifest.status = "cancelled";
+			run.manifest.endedAt = endedAt.toISOString();
+			run.summary.status = "cancelled";
+			run.summary.totalDurationMs = endedAt.getTime() - new Date(run.manifest.startedAt).getTime();
+			params.cancelRunningRun(run);
+		}
+		params.runs.delete(runId);
+		await rm(run.runDir, {
+			recursive: true,
+			force: true
+		});
+		deletedRuns += 1;
+	}
+	return deletedRuns;
+}
 async function recomputeEvalStatusesInRuns(params) {
 	let updatedRuns = 0;
 	for (const run of params.runs) {
@@ -6384,6 +6554,12 @@ function encodeCaseDetailFileName(caseId) {
 	return encodeURIComponent(caseId);
 }
 //#endregion
+//#region ../runner/src/stackFormatting.ts
+const orphanedAnsiSgrPattern = /\[(?:\d{1,3}(?:;\d{1,3})*)?m/g;
+function stripTerminalControlCodes(value) {
+	return stripVTControlCharacters(value).replaceAll(orphanedAnsiSgrPattern, "");
+}
+//#endregion
 //#region ../runner/src/moduleIsolation.ts
 const isolationParam = "agent-evals-isolate";
 const pathSegmentSeparatorPattern = /[\\/]+/;
@@ -6474,12 +6650,6 @@ async function runWithModuleIsolation(context, fn) {
 	return await isolationStorage.run(context, fn);
 }
 //#endregion
-//#region ../runner/src/stackFormatting.ts
-const orphanedAnsiSgrPattern = /\[(?:\d{1,3}(?:;\d{1,3})*)?m/g;
-function stripTerminalControlCodes(value) {
-	return stripVTControlCharacters(value).replaceAll(orphanedAnsiSgrPattern, "");
-}
-//#endregion
 //#region ../runner/src/runExecution.ts
 function filterEvalCases(cases, caseIds) {
 	if (!caseIds || caseIds.length === 0) return cases;
@@ -7061,6 +7231,8 @@ async function executeRun({ runState, request, runDir, config, cacheStore, lastR
 							const duplicateCaseIds = findDuplicateCaseIds(runnableCases);
 							if (duplicateCaseIds.length > 0) throw new Error(`Duplicate case id${duplicateCaseIds.length === 1 ? "" : "s"} in ${evalMeta.filePath}#${evalMeta.id}: ${duplicateCaseIds.join(", ")}`);
 							const cases = filterEvalCases(runnableCases, request.target.caseIds);
+							evalMeta.caseCount = runnableCases.length;
+							evalMeta.caseIds = runnableCases.map((evalCase) => evalCase.id);
 							runState.summary.totalCases += cases.length;
 							const defaultConfig = resolveEvalDefaultConfig({
 								evalDef,
@@ -7269,4 +7441,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { apiCallMetricSchema as $, getCurrentScope as $n, cacheDebugKeyEntrySchema as $t, createRunRequestSchema as A, repoFileRefSchema as An, runLogEntrySchema as At, getNestedAttribute as B, deserializeCacheValue as Bn, manualInputNumberFieldSchema as Bt, loadConfig as C, cellValueSchema as Cn, caseRowSchema as Ct, createFsCacheStore as D, fileRefSchema as Dn, evalStatItemSchema as Dt, validateCharts as E, columnKindSchema as En, evalStatAggregateSchema as Et, extractApiCalls as F, evalSpan as Fn, manualInputBooleanFieldSchema as Ft, deriveStatusFromChildStatuses as G, readManualInputFile as Gn, evalChartAxisSchema as Gt, getEvalDisplayStatus as H, serializeCacheValue as Hn, manualInputSelectOptionSchema as Ht, extractLlmCalls as I, evalTracer as In, manualInputDescriptorSchema as It, DEFAULT_API_CALLS_CONFIG as J, advanceEvalTime as Jn, evalChartConfigSchema as Jt, runManifestSchema as K, evalExpect as Kn, evalChartBuiltinMetricSchema as Kt, simulateLlmCallCost as L, hashCacheKey as Ln, manualInputFieldDescriptorSchema as Lt, sseEnvelopeSchema as M, z$1 as Mn, runLogLocationSchema as Mt, extractCacheEntries as N, buildTraceTree as Nn, runLogPhaseSchema as Nt, configReloadStateSchema as O, jsonCellSchema as On, evalStatsConfigSchema as Ot, extractCacheHits as P, captureEvalSpanError as Pn, scoreTraceSchema as Pt, apiCallMetricPlacementSchema as Q, evalLog as Qn, evalChartsConfigSchema as Qt, simulateTokenAllocation as R, hashCacheKeySync as Rn, manualInputJsonFieldSchema as Rt, resolveEvalDefaultConfig as S, traceSpanWarningSchema as Sn, caseDetailSchema as St, normalizeScoreDef as T, columnFormatSchema as Tn, evalFreshnessStatusSchema as Tt, deriveScopedSummaryFromCases as U, repoFile as Un, manualInputTextFieldSchema as Ut, getEvalTitle as V, serializeCacheRecording as Vn, manualInputSelectFieldSchema as Vt, deriveStatusFromCaseRows as W, manualInputFileValueSchema as Wn, evalChartAggregateSchema as Wt, agentEvalsConfigSchema as X, configureEvalRunLogs as Xn, evalChartTooltipExtraSchema as Xt, DEFAULT_LLM_CALLS_CONFIG as Y, appendToEvalOutput as Yn, evalChartMetricSchema as Yt, apiCallMetricFormatSchema as Z, evalAssert as Zn, evalChartTypeSchema as Zt, buildManualInputDescriptor as _, traceDisplayConfigSchema as _n, buildCaseKey as _t, getLastRunStatuses as a, cacheModeSchema as an, nextEvalId as ar, llmCallCostCurrencySchema as at, loadEvalModule as b, traceSpanKindSchema as bn, getCaseRowEvalKey as bt, loadPersistedRunSnapshots as c, cacheRecordingSchema as cn, runInExistingEvalScope as cr, llmCallMetricSchema as ct, persistRunState as d, spanCacheOptionsSchema as dn, startEvalBackgroundJob as dr, llmCallsConfigSchema as dt, cacheDebugKeyFileSchema as en, getEvalCaseInput as er, apiCallsConfigSchema as et, recomputeEvalStatusesInRuns as f, traceCacheRefSchema as fn, defineEval as fr, removeDefaultConfigSchema as ft, resolveArtifactPath as g, traceAttributeDisplaySchema as gn, trialSelectionModeSchema as gt, resolveTracePresentation as h, traceAttributeDisplayPlacementSchema as hn, runLogsConfigSchema as ht, generateRunId as i, cacheListItemSchema as in, mergeEvalOutput as ir, evalDeriveConfigSchema as it, updateManualScoreRequestSchema as j, runArtifactRefSchema as jn, runLogLevelSchema as jt, configReloadStatusSchema as k, numberDisplayOptionsSchema as kn, evalSummarySchema as kt, nextShortIdFromSnapshots as l, cacheStatusSchema as ln, setEvalOutput as lr, llmCallPricingRateSchema as lt, runTouchesEval as m, traceAttributeDisplayInputSchema as mn, resolveLlmCallsConfig as mt, getTargetEvalKeys as n, cacheEntryWithDebugKeySchema as nn, incrementEvalOutput as nr, evalColumnOverrideSchema as nt, getLatestRunInfos as o, cacheOperationTypeSchema as on, runInEvalRuntimeScope as or, llmCallMetricFormatSchema as ot, recomputePersistedCaseStatus as p, traceAttributeDisplayFormatSchema as pn, getEvalRegistry as pr, resolveApiCallsConfig as pt, runSummarySchema as q, EvalAssertionError as qn, evalChartColorSchema as qt, getTargetEvals as r, cacheFileSchema as rn, isInEvalScope as rr, evalColumnsSchema as rt, loadPersistedRunSnapshot as s, cacheRecordingOpSchema as sn, runInEvalScope as sr, llmCallMetricPlacementSchema as st, executeRun as t, cacheEntrySchema as tn, getEvalStartTime as tr, defaultConfigKeySchema as tt, persistCaseDetail as u, serializedCacheSpanSchema as un, setScopeCacheContext as ur, llmCallPricingSchema as ut, parseManualInputValues as v, traceDisplayInputConfigSchema as vn, buildEvalKey as vt, buildDeclaredColumnDefs as w, columnDefSchema as wn, discoveryIssueSchema as wt, parseEvalDiscovery as x, traceSpanSchema as xn, assertionFailureSchema as xt, deriveEvalFreshness as y, traceSpanErrorSchema as yn, getCaseRowCaseKey as yt, applyDerivedCallAttributes as z, deserializeCacheRecording as zn, manualInputMultilineFieldSchema as zt };
+export { apiCallMetricFormatSchema as $, evalAssert as $n, evalChartTypeSchema as $t, configReloadStateSchema as A, jsonCellSchema as An, evalStatsConfigSchema as At, simulateTokenAllocation as B, hashCacheKeySync as Bn, manualInputJsonFieldSchema as Bt, parseEvalDiscovery as C, traceSpanSchema as Cn, assertionFailureSchema as Ct, normalizeScoreDef as D, columnFormatSchema as Dn, evalFreshnessStatusSchema as Dt, buildDeclaredColumnDefs as E, columnDefSchema as En, discoveryIssueSchema as Et, extractCacheEntries as F, buildTraceTree as Fn, runLogPhaseSchema as Ft, deriveScopedSummaryFromCases as G, repoFile as Gn, manualInputTextFieldSchema as Gt, getNestedAttribute as H, deserializeCacheValue as Hn, manualInputNumberFieldSchema as Ht, extractCacheHits as I, captureEvalSpanError as In, scoreTraceSchema as It, runManifestSchema as J, evalExpect as Jn, evalChartBuiltinMetricSchema as Jt, deriveStatusFromCaseRows as K, manualInputFileValueSchema as Kn, evalChartAggregateSchema as Kt, extractApiCalls as L, evalSpan as Ln, manualInputBooleanFieldSchema as Lt, createRunRequestSchema as M, repoFileRefSchema as Mn, runLogEntrySchema as Mt, updateManualScoreRequestSchema as N, runArtifactRefSchema as Nn, runLogLevelSchema as Nt, validateCharts as O, columnKindSchema as On, evalStatAggregateSchema as Ot, sseEnvelopeSchema as P, z$1 as Pn, runLogLocationSchema as Pt, agentEvalsConfigSchema as Q, configureEvalRunLogs as Qn, evalChartTooltipExtraSchema as Qt, extractLlmCalls as R, evalTracer as Rn, manualInputDescriptorSchema as Rt, loadEvalModule as S, traceSpanKindSchema as Sn, getCaseRowEvalKey as St, loadConfig as T, cellValueSchema as Tn, caseRowSchema as Tt, getEvalTitle as U, serializeCacheRecording as Un, manualInputSelectFieldSchema as Ut, applyDerivedCallAttributes as V, deserializeCacheRecording as Vn, manualInputMultilineFieldSchema as Vt, getEvalDisplayStatus as W, serializeCacheValue as Wn, manualInputSelectOptionSchema as Wt, DEFAULT_API_CALLS_CONFIG as X, advanceEvalTime as Xn, evalChartConfigSchema as Xt, runSummarySchema as Y, EvalAssertionError as Yn, evalChartColorSchema as Yt, DEFAULT_LLM_CALLS_CONFIG as Z, appendToEvalOutput as Zn, evalChartMetricSchema as Zt, resolveTracePresentation as _, traceAttributeDisplayPlacementSchema as _n, runLogsConfigSchema as _t, generateRunId as a, cacheFileSchema as an, isInEvalScope as ar, evalColumnsSchema as at, parseManualInputValues as b, traceDisplayInputConfigSchema as bn, buildEvalKey as bt, loadPersistedRunSnapshot as c, cacheOperationTypeSchema as cn, runInEvalRuntimeScope as cr, llmCallMetricFormatSchema as ct, persistCaseDetail as d, cacheStatusSchema as dn, setEvalOutput as dr, llmCallPricingRateSchema as dt, evalChartsConfigSchema as en, evalLog as er, apiCallMetricPlacementSchema as et, deleteTemporaryRuns as f, serializedCacheSpanSchema as fn, setScopeCacheContext as fr, llmCallPricingSchema as ft, runTouchesEval as g, traceAttributeDisplayInputSchema as gn, resolveLlmCallsConfig as gt, recomputePersistedCaseStatus as h, traceAttributeDisplayFormatSchema as hn, getEvalRegistry as hr, resolveApiCallsConfig as ht, stripTerminalControlCodes as i, cacheEntryWithDebugKeySchema as in, incrementEvalOutput as ir, evalColumnOverrideSchema as it, configReloadStatusSchema as j, numberDisplayOptionsSchema as jn, evalSummarySchema as jt, createFsCacheStore as k, fileRefSchema as kn, evalStatItemSchema as kt, loadPersistedRunSnapshots as l, cacheRecordingOpSchema as ln, runInEvalScope as lr, llmCallMetricPlacementSchema as lt, recomputeEvalStatusesInRuns as m, traceCacheRefSchema as mn, defineEval as mr, removeDefaultConfigSchema as mt, getTargetEvalKeys as n, cacheDebugKeyFileSchema as nn, getEvalCaseInput as nr, apiCallsConfigSchema as nt, getLastRunStatuses as o, cacheListItemSchema as on, mergeEvalOutput as or, evalDeriveConfigSchema as ot, persistRunState as p, spanCacheOptionsSchema as pn, startEvalBackgroundJob as pr, llmCallsConfigSchema as pt, deriveStatusFromChildStatuses as q, readManualInputFile as qn, evalChartAxisSchema as qt, getTargetEvals as r, cacheEntrySchema as rn, getEvalStartTime as rr, defaultConfigKeySchema as rt, getLatestRunInfos as s, cacheModeSchema as sn, nextEvalId as sr, llmCallCostCurrencySchema as st, executeRun as t, cacheDebugKeyEntrySchema as tn, getCurrentScope as tr, apiCallMetricSchema as tt, nextShortIdFromSnapshots as u, cacheRecordingSchema as un, runInExistingEvalScope as ur, llmCallMetricSchema as ut, resolveArtifactPath as v, traceAttributeDisplaySchema as vn, trialSelectionModeSchema as vt, resolveEvalDefaultConfig as w, traceSpanWarningSchema as wn, caseDetailSchema as wt, deriveEvalFreshness as x, traceSpanErrorSchema as xn, getCaseRowCaseKey as xt, buildManualInputDescriptor as y, traceDisplayConfigSchema as yn, buildCaseKey as yt, simulateLlmCallCost as z, hashCacheKey as zn, manualInputFieldDescriptorSchema as zt };