@ls-stack/agent-eval 0.16.1 → 0.17.0

package/dist/{runOrchestration-BDyNrRQT.mjs → runOrchestration-D2okEB3I.mjs}

@@ -4,6 +4,7 @@ import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/p
 import { extname, isAbsolute, join, relative, resolve } from "node:path";
 import { z, z as z$1 } from "zod/v4";
 import { AsyncLocalStorage } from "node:async_hooks";
+import { formatWithOptions } from "node:util";
 import { Buffer as Buffer$1 } from "node:buffer";
 import { getCompositeKey } from "@ls-stack/utils/getCompositeKey";
 import { existsSync } from "node:fs";
@@ -49,6 +50,25 @@ const scopeStorage = new AsyncLocalStorage();
 const runtimeScopeStorage = new AsyncLocalStorage();
 let activeEvalScopeCount = 0;
 let activeEvalRuntimeScopeCount = 0;
+let consoleCaptureEnabled = true;
+const maxLogMessageLength = 2e4;
+const maxLogStringLength = 1e4;
+const maxLogArrayLength = 100;
+const maxLogObjectEntries = 100;
+const maxLogValueDepth = 5;
+const consoleCaptureMethods = [
+	"log",
+	"info",
+	"warn",
+	"error"
+];
+const runtimeConsole = globalThis.console;
+const originalConsoleMethods = {
+	log: runtimeConsole.log.bind(runtimeConsole),
+	info: runtimeConsole.info.bind(runtimeConsole),
+	warn: runtimeConsole.warn.bind(runtimeConsole),
+	error: runtimeConsole.error.bind(runtimeConsole)
+};
 /** Error thrown when an eval assertion fails during case execution. */
 var EvalAssertionError = class extends Error {
 	constructor(message) {
@@ -73,6 +93,155 @@ function isInEvalScope() {
 	if (activeEvalRuntimeScopeCount === 0) return null;
 	return runtimeScopeStorage.getStore() ?? null;
 }
+function normalizeLogLevel(level) {
+	return level === "warning" ? "warn" : level;
+}
+function getCurrentLogPhase() {
+	const runtimeScope = runtimeScopeStorage.getStore();
+	if (runtimeScope === "eval" || runtimeScope === "derive" || runtimeScope === "outputsSchema" || runtimeScope === "scorer") return runtimeScope;
+	return null;
+}
+function formatLogArgs(args) {
+	const formatted = formatWithOptions({
+		depth: 2,
+		maxArrayLength: 100,
+		maxStringLength: 1e4,
+		breakLength: 80,
+		compact: 3
+	}, ...args);
+	if (formatted.length <= maxLogMessageLength) return {
+		message: formatted,
+		truncated: false
+	};
+	return {
+		message: `${formatted.slice(0, maxLogMessageLength)}...`,
+		truncated: true
+	};
+}
+function truncateLogString(value, ctx) {
+	if (value.length <= maxLogStringLength) return value;
+	ctx.truncated = true;
+	return `${value.slice(0, maxLogStringLength)}...`;
+}
+function primitiveToLogValue(value, ctx) {
+	if (typeof value === "string") return {
+		handled: true,
+		value: truncateLogString(value, ctx)
+	};
+	if (value === null || typeof value === "number" || typeof value === "boolean") return {
+		handled: true,
+		value
+	};
+	if (value === void 0) return {
+		handled: true,
+		value: "[undefined]"
+	};
+	if (typeof value === "bigint") return {
+		handled: true,
+		value: `${value.toString()}n`
+	};
+	if (typeof value === "symbol") return {
+		handled: true,
+		value: String(value)
+	};
+	if (typeof value === "function") return {
+		handled: true,
+		value: `[Function${value.name.length > 0 ? `: ${value.name}` : ""}]`
+	};
+	return {
+		handled: false,
+		value: null
+	};
+}
+function objectToLogValue(value, ctx, depth) {
+	if (value instanceof Date) return value.toISOString();
+	if (value instanceof Error) return {
+		name: value.name,
+		message: value.message,
+		stack: value.stack
+	};
+	if (ctx.seen.has(value)) return "[Circular]";
+	if (depth >= maxLogValueDepth) {
+		ctx.truncated = true;
+		return Array.isArray(value) ? "[Array]" : "[Object]";
+	}
+	ctx.seen.add(value);
+	try {
+		if (Array.isArray(value)) {
+			const limited = value.slice(0, maxLogArrayLength).map((item) => toLogJsonValue(item, ctx, depth + 1));
+			if (value.length > maxLogArrayLength) {
+				ctx.truncated = true;
+				limited.push(`[... ${String(value.length - maxLogArrayLength)} more]`);
+			}
+			return limited;
+		}
+		const entries = Object.entries(value);
+		const result = {};
+		for (const [key, entryValue] of entries.slice(0, maxLogObjectEntries)) result[key] = toLogJsonValue(entryValue, ctx, depth + 1);
+		if (entries.length > maxLogObjectEntries) {
+			ctx.truncated = true;
+			result.__truncated = `${String(entries.length - maxLogObjectEntries)} more properties`;
+		}
+		return result;
+	} finally {
+		ctx.seen.delete(value);
+	}
+}
+function toLogJsonValue(value, ctx, depth) {
+	const primitive = primitiveToLogValue(value, ctx);
+	if (primitive.handled) return primitive.value;
+	if (typeof value === "object" && value !== null) return objectToLogValue(value, ctx, depth);
+	return String(value);
+}
+function toLogJsonArgs(args) {
+	const ctx = {
+		seen: /* @__PURE__ */ new WeakSet(),
+		truncated: false
+	};
+	return {
+		args: args.map((value) => toLogJsonValue(value, ctx, 0)),
+		truncated: ctx.truncated
+	};
+}
+function recordEvalLog(level, args) {
+	const scope = getCurrentScope();
+	const phase = getCurrentLogPhase();
+	if (!scope || !phase) return;
+	const preview = formatLogArgs(args);
+	const jsonArgs = toLogJsonArgs(args);
+	scope.logs.push({
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		level: normalizeLogLevel(level),
+		phase,
+		message: preview.message,
+		args: jsonArgs.args,
+		truncated: preview.truncated || jsonArgs.truncated
+	});
+}
+for (const method of consoleCaptureMethods) runtimeConsole[method] = (...args) => {
+	if (consoleCaptureEnabled) recordEvalLog(method, args);
+	originalConsoleMethods[method](...args);
+};
+/**
+* Configure whether console methods are captured as eval case logs.
+*
+* Runner-internal helper. When disabled, console output still prints normally;
+* only automatic persistence to `caseDetail.logs` is skipped. Manual
+* `evalLog(...)` calls are unaffected.
+*/
+function configureEvalRunLogs(options) {
+	consoleCaptureEnabled = options.captureConsole;
+}
+/**
+* Record a manual log entry on the active eval case.
+*
+* Values are formatted with Node-style console formatting and capped before
+* persistence so a single log cannot make run artifacts unbounded. Calls made
+* outside active case-owned eval phases are ignored.
+*/
+function evalLog(level, ...args) {
+	recordEvalLog(level, args);
+}
 function registerBackgroundJobInScope(scope, promise) {
 	const trackedPromise = promise.then(() => {
 		scope.pendingBackgroundJobs.delete(trackedPromise);
@@ -164,6 +333,7 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		input: options.input,
 		outputs: {},
 		assertionFailures: [],
+		logs: [],
 		spans: [],
 		checkpoints: /* @__PURE__ */ new Map(),
 		spanStack: [],
@@ -332,107 +502,6 @@ function evalAssert(condition, message) {
 	throw error;
 }
 //#endregion
-//#region ../sdk/src/cacheKey.ts
-var SerializedCacheKeyValue = class {
-	value;
-	constructor(value) {
-		this.value = value;
-	}
-};
-/**
-* Hash the components of a cache key into a deterministic hex digest.
-*
-* Native `Blob` and `File` values use stable metadata by default. Pass
-* `serializeFileBytes: true` to read them asynchronously and include their byte
-* hash in the key.
-*/
-async function hashCacheKey(input, options = {}) {
-	return hashCacheKeySyncMaterialized(options.serializeFileBytes === true ? await materializeAsyncCacheKeyValue(input) : input);
-}
-/**
-* Synchronously hash cache key components. This supports JSON-like data and
-* in-memory binary values such as `Buffer`, `ArrayBuffer`, and typed arrays,
-* plus stable metadata for native `Blob` and `File` values.
-*/
-function hashCacheKeySync(input) {
-	return hashCacheKeySyncMaterialized(input);
-}
-function hashCacheKeySyncMaterialized(input) {
-	return createHash("sha256").update(getCompositeKey(input, { stringify: stringifyCacheKeyValue })).digest("hex");
-}
-function stringifyCacheKeyValue(value) {
-	if (value instanceof SerializedCacheKeyValue) return value.value;
-	if (Buffer$1.isBuffer(value)) return `$buffer:${hashBytes(value)}`;
-	if (isArrayBuffer(value)) return `$arrayBuffer:${hashBytes(new Uint8Array(value))}`;
-	if (isSharedArrayBuffer(value)) return `$sharedArrayBuffer:${hashBytes(new Uint8Array(value))}`;
-	if (isArrayBufferView(value)) {
-		const bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
-		return `$${value.constructor.name}:${hashBytes(bytes)}`;
-	}
-	if (isFile$1(value)) return `$file:${getCompositeKey({
-		lastModified: value.lastModified,
-		name: value.name,
-		size: value.size,
-		type: value.type
-	})}`;
-	if (isBlob$1(value)) return `$blob:${getCompositeKey({
-		size: value.size,
-		type: value.type
-	})}`;
-}
-async function materializeAsyncCacheKeyValue(value, refs = /* @__PURE__ */ new WeakSet()) {
-	const serialized = await stringifyAsyncCacheKeyValue(value);
-	if (serialized !== void 0) return new SerializedCacheKeyValue(serialized);
-	if (stringifyCacheKeyValue(value) !== void 0) return value;
-	if (!value || typeof value !== "object") return value;
-	if (Array.isArray(value)) {
-		const items = [];
-		for (const item of value) items.push(await materializeAsyncCacheKeyValue(item, refs));
-		return items;
-	}
-	if (refs.has(value)) throw new Error("Circular reference detected");
-	refs.add(value);
-	const entries = [];
-	for (const [key, entryValue] of Object.entries(value)) entries.push([key, await materializeAsyncCacheKeyValue(entryValue, refs)]);
-	refs.delete(value);
-	return Object.fromEntries(entries);
-}
-async function stringifyAsyncCacheKeyValue(value) {
-	if (isFile$1(value)) return `$file:${getCompositeKey({
-		bytes: await hashBlobBytes(value),
-		lastModified: value.lastModified,
-		name: value.name,
-		size: value.size,
-		type: value.type
-	})}`;
-	if (isBlob$1(value)) return `$blob:${getCompositeKey({
-		bytes: await hashBlobBytes(value),
-		size: value.size,
-		type: value.type
-	})}`;
-}
-async function hashBlobBytes(value) {
-	return hashBytes(new Uint8Array(await value.arrayBuffer()));
-}
-function hashBytes(value) {
-	return createHash("sha256").update(value).digest("hex");
-}
-function isArrayBuffer(value) {
-	return value instanceof ArrayBuffer;
-}
-function isSharedArrayBuffer(value) {
-	return value instanceof SharedArrayBuffer;
-}
-function isArrayBufferView(value) {
-	return ArrayBuffer.isView(value);
-}
-function isBlob$1(value) {
-	return value instanceof Blob;
-}
-function isFile$1(value) {
-	return value instanceof File;
-}
-//#endregion
 //#region ../../node_modules/.pnpm/seroval@1.5.2/node_modules/seroval/dist/esm/production/index.mjs
 var L$1 = ((i) => (i[i.AggregateError = 1] = "AggregateError", i[i.ArrowFunction = 2] = "ArrowFunction", i[i.ErrorPrototypeStack = 4] = "ErrorPrototypeStack", i[i.ObjectAssign = 8] = "ObjectAssign", i[i.BigIntTypedArray = 16] = "BigIntTypedArray", i[i.RegExp = 32] = "RegExp", i))(L$1 || {});
 var v$1 = Symbol.asyncIterator, mr = Symbol.hasInstance, R = Symbol.isConcatSpreadable, C = Symbol.iterator, pr = Symbol.match, dr = Symbol.matchAll, gr = Symbol.replace, yr = Symbol.search, Nr = Symbol.species, br = Symbol.split, vr = Symbol.toPrimitive, P$1 = Symbol.toStringTag, Cr = Symbol.unscopables, ve = {
@@ -2223,6 +2292,107 @@ function deserializeCacheRecording(recording) {
 	};
 }
 //#endregion
+//#region ../sdk/src/cacheKey.ts
+var SerializedCacheKeyValue = class {
+	value;
+	constructor(value) {
+		this.value = value;
+	}
+};
+/**
+* Hash the components of a cache key into a deterministic hex digest.
+*
+* Native `Blob` and `File` values use stable metadata by default. Pass
+* `serializeFileBytes: true` to read them asynchronously and include their byte
+* hash in the key.
+*/
+async function hashCacheKey(input, options = {}) {
+	return hashCacheKeySyncMaterialized(options.serializeFileBytes === true ? await materializeAsyncCacheKeyValue(input) : input);
+}
+/**
+* Synchronously hash cache key components. This supports JSON-like data and
+* in-memory binary values such as `Buffer`, `ArrayBuffer`, and typed arrays,
+* plus stable metadata for native `Blob` and `File` values.
+*/
+function hashCacheKeySync(input) {
+	return hashCacheKeySyncMaterialized(input);
+}
+function hashCacheKeySyncMaterialized(input) {
+	return createHash("sha256").update(getCompositeKey(input, { stringify: stringifyCacheKeyValue })).digest("hex");
+}
+function stringifyCacheKeyValue(value) {
+	if (value instanceof SerializedCacheKeyValue) return value.value;
+	if (Buffer$1.isBuffer(value)) return `$buffer:${hashBytes(value)}`;
+	if (isArrayBuffer(value)) return `$arrayBuffer:${hashBytes(new Uint8Array(value))}`;
+	if (isSharedArrayBuffer(value)) return `$sharedArrayBuffer:${hashBytes(new Uint8Array(value))}`;
+	if (isArrayBufferView(value)) {
+		const bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+		return `$${value.constructor.name}:${hashBytes(bytes)}`;
+	}
+	if (isFile$1(value)) return `$file:${getCompositeKey({
+		lastModified: value.lastModified,
+		name: value.name,
+		size: value.size,
+		type: value.type
+	})}`;
+	if (isBlob$1(value)) return `$blob:${getCompositeKey({
+		size: value.size,
+		type: value.type
+	})}`;
+}
+async function materializeAsyncCacheKeyValue(value, refs = /* @__PURE__ */ new WeakSet()) {
+	const serialized = await stringifyAsyncCacheKeyValue(value);
+	if (serialized !== void 0) return new SerializedCacheKeyValue(serialized);
+	if (stringifyCacheKeyValue(value) !== void 0) return value;
+	if (!value || typeof value !== "object") return value;
+	if (Array.isArray(value)) {
+		const items = [];
+		for (const item of value) items.push(await materializeAsyncCacheKeyValue(item, refs));
+		return items;
+	}
+	if (refs.has(value)) throw new Error("Circular reference detected");
+	refs.add(value);
+	const entries = [];
+	for (const [key, entryValue] of Object.entries(value)) entries.push([key, await materializeAsyncCacheKeyValue(entryValue, refs)]);
+	refs.delete(value);
+	return Object.fromEntries(entries);
+}
+async function stringifyAsyncCacheKeyValue(value) {
+	if (isFile$1(value)) return `$file:${getCompositeKey({
+		bytes: await hashBlobBytes(value),
+		lastModified: value.lastModified,
+		name: value.name,
+		size: value.size,
+		type: value.type
+	})}`;
+	if (isBlob$1(value)) return `$blob:${getCompositeKey({
+		bytes: await hashBlobBytes(value),
+		size: value.size,
+		type: value.type
+	})}`;
+}
+async function hashBlobBytes(value) {
+	return hashBytes(new Uint8Array(await value.arrayBuffer()));
+}
+function hashBytes(value) {
+	return createHash("sha256").update(value).digest("hex");
+}
+function isArrayBuffer(value) {
+	return value instanceof ArrayBuffer;
+}
+function isSharedArrayBuffer(value) {
+	return value instanceof SharedArrayBuffer;
+}
+function isArrayBufferView(value) {
+	return ArrayBuffer.isView(value);
+}
+function isBlob$1(value) {
+	return value instanceof Blob;
+}
+function isFile$1(value) {
+	return value instanceof File;
+}
+//#endregion
 //#region ../sdk/src/cacheRecording.ts
 function mergeSpanAttributes$1(span, attributes) {
 	span.attributes = {
@@ -2571,6 +2741,11 @@ function createTraceCache(generateSpanId) {
 				storedAt: (/* @__PURE__ */ new Date()).toISOString(),
 				codeFingerprint: cacheCtx.codeFingerprint,
 				recording: await serializeCacheRecording(recording)
+			}, {
+				rawKey: info.key,
+				operationType: "value",
+				operationName: info.name,
+				codeFingerprint: cacheCtx.codeFingerprint
 			});
 		}
 		return bodyResult;
@@ -2996,7 +3171,12 @@ async function traceSpanInternal(info, fn) {
 					codeFingerprint: ctx.codeFingerprint,
 					recording: await serializeCacheRecording(recording)
 				};
-				await ctx.adapter.write(entry);
+				await ctx.adapter.write(entry, {
+					rawKey: cacheOpts.key,
+					operationType: "span",
+					operationName: info.name,
+					codeFingerprint: ctx.codeFingerprint
+				});
 			}
 			return bodyResult;
 		}
@@ -3415,12 +3595,31 @@ const cacheEntrySchema = z.object({
 	codeFingerprint: z.string(),
 	recording: cacheRecordingSchema
 });
+/** Debug-only raw key metadata stored outside the reusable cache entry. */
+const cacheDebugKeyEntrySchema = z.object({
+	version: z.literal(1),
+	key: z.string(),
+	namespace: z.string(),
+	operationType: cacheOperationTypeSchema,
+	operationName: z.string(),
+	storedAt: z.string(),
+	codeFingerprint: z.string(),
+	rawKey: z.unknown()
+});
+/** Cache lookup response with optional debug-only raw key data. */
+const cacheEntryWithDebugKeySchema = cacheEntrySchema.extend({ debugKey: cacheDebugKeyEntrySchema.optional() });
 /** Persisted per-owner cache file containing multiple cache entries. */
 const cacheFileSchema = z.object({
 	version: z.literal(1),
 	owner: z.string(),
 	entries: z.record(z.string(), cacheEntrySchema)
 });
+/** Persisted per-owner debug file containing raw cache key metadata. */
+const cacheDebugKeyFileSchema = z.object({
+	version: z.literal(1),
+	owner: z.string(),
+	entries: z.record(z.string(), cacheDebugKeyEntrySchema)
+});
 //#endregion
 //#region ../shared/src/schemas/chart.ts
 /** Chart type rendered for a single eval history chart. */
@@ -3623,6 +3822,40 @@ const assertionFailureSchema = z.object({
 	stack: z.string().optional()
 });
 const legacyAssertionFailureSchema = z.string().transform((message) => ({ message }));
+/** Severity level for one log captured during a case run. */
+const runLogLevelSchema = z.enum([
+	"log",
+	"info",
+	"warn",
+	"error"
+]);
+/** Eval runner phase that emitted a captured case log. */
+const runLogPhaseSchema = z.enum([
+	"eval",
+	"derive",
+	"outputsSchema",
+	"scorer"
+]);
+/** Schema for one persisted log entry captured during a case run. */
+const runLogEntrySchema = z.object({
+	/** ISO timestamp for when the log was captured. */
+	timestamp: z.string(),
+	/** Normalized log level. */
+	level: runLogLevelSchema,
+	/** Case-owned runner phase that emitted the log. */
+	phase: runLogPhaseSchema,
+	/** Human-readable preview formatted from the original log arguments. */
+	message: z.string(),
+	/** JSON-safe captured log arguments rendered in the UI. */
+	args: z.array(z.unknown()).default([]),
+	/** Whether `message` was capped before persistence. */
+	truncated: z.boolean().default(false),
+	/**
+	* Optional source label for logs emitted from a nested case-owned activity,
+	* such as a score key.
+	*/
+	source: z.string().optional()
+});
 /** Trace payload captured while computing one score for a case. */
 const scoreTraceSchema = z.object({
 	trace: z.array(traceSpanSchema),
@@ -3651,6 +3884,8 @@ const caseDetailSchema = z.object({
 	scoringTraces: z.record(z.string(), scoreTraceSchema).optional(),
 	columns: z.record(z.string(), cellValueSchema),
 	assertionFailures: z.array(z.union([assertionFailureSchema, legacyAssertionFailureSchema])),
+	/** Logs captured from manual `evalLog(...)` calls and enabled console calls. */
+	logs: z.array(runLogEntrySchema).default([]),
 	error: z.object({
 		name: z.string().optional(),
 		message: z.string(),
@@ -3802,6 +4037,14 @@ const apiCallsConfigSchema = z.object({
 	/** Custom user-defined metrics surfaced on each API call. */
 	metrics: z.array(apiCallMetricSchema).optional()
 });
+/** Schema for workspace-level run log capture options. */
+const runLogsConfigSchema = z.object({ 
+/**
+* Capture `console.log`, `console.info`, `console.warn`, and
+* `console.error` calls made inside active eval case scopes. Defaults to
+* `true`; manual `evalLog(...)` calls are always captured.
+*/
+captureConsole: z.boolean().optional() });
 /** Default LLM-calls config the UI uses before the workspace fetch resolves. */
 const DEFAULT_LLM_CALLS_CONFIG = {
 	kinds: ["llm"],
@@ -3917,6 +4160,7 @@ const agentEvalsConfigSchema = z.object({
 	traceDisplay: traceDisplayInputConfigSchema.optional(),
 	llmCalls: llmCallsConfigSchema.optional(),
 	apiCalls: apiCallsConfigSchema.optional(),
+	runLogs: runLogsConfigSchema.optional(),
 	cache: z.object({
 		enabled: z.boolean().optional(),
 		dir: z.string().optional(),
@@ -4490,15 +4734,33 @@ const defaultMaxEntriesPerNamespace = 100;
 */
 function createFsCacheStore(options) {
 	const cacheDir = resolve(options.workspaceRoot, options.dir ?? ".agent-evals/cache");
+	const debugDir = resolve(options.workspaceRoot, options.debugDir ?? ".agent-evals/cache-debug");
 	const defaultMaxEntries = normalizeMaxEntries(options.maxEntriesPerNamespace);
 	return {
 		dir() {
 			return cacheDir;
 		},
+		debugDir() {
+			return debugDir;
+		},
 		async lookup(namespace, keyHash) {
 			return (await readCacheFile(cacheDir, ownerFromNamespace(namespace)))?.entries[keyHash] ?? null;
 		},
-		async write(entry) {
+		async lookupWithDebug(namespace, keyHash) {
+			const owner = ownerFromNamespace(namespace);
+			const entry = (await readCacheFile(cacheDir, owner))?.entries[keyHash] ?? null;
+			if (entry === null) return null;
+			const debugKey = (await readDebugKeyFile(debugDir, owner))?.entries[keyHash];
+			const deserializedEntry = {
+				...entry,
+				recording: deserializeCacheRecording(entry.recording)
+			};
+			return debugKey === void 0 ? deserializedEntry : {
+				...deserializedEntry,
+				debugKey
+			};
+		},
+		async write(entry, debugKey) {
 			const owner = ownerFromNamespace(entry.namespace);
 			const filePath = ownerPath(cacheDir, owner);
 			await mkdir(cacheDir, { recursive: true });
@@ -4512,6 +4774,17 @@ function createFsCacheStore(options) {
 					}, entry.namespace, maxEntriesForNamespace(entry.namespace, defaultMaxEntries, options.maxEntriesByNamespace), entry.key)
 				});
 			});
+			if (debugKey !== void 0) {
+				if ((await resultify(() => writeDebugKeyEntry({
+					debugDir,
+					entry,
+					debugKey,
+					maxEntries: maxEntriesForNamespace(entry.namespace, defaultMaxEntries, options.maxEntriesByNamespace)
+				}))).error) await resultify(() => clearDebugEntries(debugDir, {
+					namespace: entry.namespace,
+					key: entry.key
+				}));
+			}
 		},
 		async list() {
 			if (!existsSync(cacheDir)) return [];
@@ -4544,17 +4817,21 @@ function createFsCacheStore(options) {
 			return items;
 		},
 		async clear(filter) {
-			if (!existsSync(cacheDir)) return;
 			if (!filter || filter.namespace === void 0 && filter.key === void 0) {
 				await rm(cacheDir, {
 					recursive: true,
 					force: true
 				});
+				await rm(debugDir, {
+					recursive: true,
+					force: true
+				});
 				return;
 			}
 			if (filter.namespace !== void 0) {
 				const owner = ownerFromNamespace(filter.namespace);
-				await withCacheFileLock(ownerPath(cacheDir, owner), async () => {
+				const filePath = ownerPath(cacheDir, owner);
+				if (existsSync(cacheDir)) await withCacheFileLock(filePath, async () => {
 					const cacheFile = await readCacheFile(cacheDir, owner);
 					if (cacheFile === null) return;
 					await writeOrRemoveCacheFile(cacheDir, {
@@ -4566,23 +4843,27 @@ function createFsCacheStore(options) {
 						}))
 					});
 				});
+				await clearDebugEntries(debugDir, filter);
 				return;
 			}
-			const files = await readdir(cacheDir);
-			for (const fileName of files) {
-				if (!fileName.endsWith(".json")) continue;
-				const filePath = join(cacheDir, fileName);
-				await withCacheFileLock(filePath, async () => {
-					const cacheFile = await readCacheFilePath(filePath);
-					if (cacheFile === null) return;
-					const entries = Object.fromEntries(Object.entries(cacheFile.entries).filter(([key]) => key !== filter.key));
-					await writeOrRemoveCacheFile(cacheDir, {
-						version: 1,
-						owner: cacheFile.owner,
-						entries
+			if (existsSync(cacheDir)) {
+				const files = await readdir(cacheDir);
+				for (const fileName of files) {
+					if (!fileName.endsWith(".json")) continue;
+					const filePath = join(cacheDir, fileName);
+					await withCacheFileLock(filePath, async () => {
+						const cacheFile = await readCacheFilePath(filePath);
+						if (cacheFile === null) return;
+						const entries = Object.fromEntries(Object.entries(cacheFile.entries).filter(([key]) => key !== filter.key));
+						await writeOrRemoveCacheFile(cacheDir, {
+							version: 1,
+							owner: cacheFile.owner,
+							entries
+						});
 					});
-				});
+				}
 			}
+			await clearDebugEntries(debugDir, filter);
 		}
 	};
 }
@@ -4598,18 +4879,21 @@ function createBufferedCacheStore(backingStore) {
 	return {
 		async lookup(namespace, keyHash) {
 			const buffered = pendingEntries.get(toPendingKey(namespace, keyHash));
-			if (buffered !== void 0) return buffered;
+			if (buffered !== void 0) return buffered.entry;
 			return backingStore.lookup(namespace, keyHash);
 		},
-		write(entry) {
-			pendingEntries.set(toPendingKey(entry.namespace, entry.key), entry);
+		write(entry, debugKey) {
+			pendingEntries.set(toPendingKey(entry.namespace, entry.key), {
+				entry,
+				debugKey
+			});
 			return Promise.resolve();
 		},
 		async commit() {
-			for (const entry of pendingEntries.values()) await backingStore.write(entry);
+			for (const pending of pendingEntries.values()) await backingStore.write(pending.entry, pending.debugKey);
 		},
 		getPendingEntries() {
-			return [...pendingEntries.values()];
+			return [...pendingEntries.values()].map((pending) => pending.entry);
 		}
 	};
 }
@@ -4661,6 +4945,94 @@ async function writeCacheFile(cacheDir, cacheFile) {
 	await writeFile(tmpPath, JSON.stringify(cacheFile, null, 2));
 	await rename(tmpPath, filePath);
 }
+async function readDebugKeyFile(debugDir, owner) {
+	return readDebugKeyFilePath(ownerPath(debugDir, owner));
+}
+async function readDebugKeyFilePath(filePath) {
+	if (!existsSync(filePath)) return null;
+	const rawResult = await resultify(() => readFile(filePath, "utf-8"));
+	if (rawResult.error) return null;
+	const json = safeJsonParse(rawResult.value);
+	if (json === null) return null;
+	const parsed = cacheDebugKeyFileSchema.safeParse(json);
+	if (!parsed.success) return null;
+	return parsed.data;
+}
+async function writeDebugKeyEntry(params) {
+	const { debugDir, entry, debugKey, maxEntries } = params;
+	const owner = ownerFromNamespace(entry.namespace);
+	const filePath = ownerPath(debugDir, owner);
+	await mkdir(debugDir, { recursive: true });
+	await withCacheFileLock(filePath, async () => {
+		const entries = (await readDebugKeyFile(debugDir, owner))?.entries ?? {};
+		const debugEntry = {
+			version: 1,
+			key: entry.key,
+			namespace: entry.namespace,
+			operationType: debugKey.operationType,
+			operationName: debugKey.operationName,
+			storedAt: entry.storedAt,
+			codeFingerprint: debugKey.codeFingerprint,
+			rawKey: debugKey.rawKey
+		};
+		await writeDebugKeyFile(debugDir, {
+			version: 1,
+			owner,
+			entries: pruneDebugKeyEntries({
+				...entries,
+				[entry.key]: debugEntry
+			}, entry.namespace, maxEntries, entry.key)
+		});
+	});
+}
+async function clearDebugEntries(debugDir, filter) {
+	if (!existsSync(debugDir)) return;
+	if (filter.namespace !== void 0) {
+		const owner = ownerFromNamespace(filter.namespace);
+		await withCacheFileLock(ownerPath(debugDir, owner), async () => {
+			const debugFile = await readDebugKeyFile(debugDir, owner);
+			if (debugFile === null) return;
+			await writeOrRemoveDebugKeyFile(debugDir, {
+				version: 1,
+				owner,
+				entries: Object.fromEntries(Object.entries(debugFile.entries).filter(([key, entry]) => {
+					if (filter.key !== void 0) return key !== filter.key;
+					return entry.namespace !== filter.namespace;
+				}))
+			});
+		});
+		return;
+	}
+	const files = await readdir(debugDir);
+	for (const fileName of files) {
+		if (!fileName.endsWith(".json")) continue;
+		const filePath = join(debugDir, fileName);
+		await withCacheFileLock(filePath, async () => {
+			const debugFile = await readDebugKeyFilePath(filePath);
+			if (debugFile === null) return;
+			const entries = Object.fromEntries(Object.entries(debugFile.entries).filter(([key]) => key !== filter.key));
+			await writeOrRemoveDebugKeyFile(debugDir, {
+				version: 1,
+				owner: debugFile.owner,
+				entries
+			});
+		});
+	}
+}
+async function writeOrRemoveDebugKeyFile(debugDir, debugFile) {
+	if (Object.keys(debugFile.entries).length === 0) {
+		await rm(ownerPath(debugDir, debugFile.owner), { force: true });
+		return;
+	}
+	await writeDebugKeyFile(debugDir, debugFile);
+}
+async function writeDebugKeyFile(debugDir, debugFile) {
+	await mkdir(debugDir, { recursive: true });
+	const filePath = ownerPath(debugDir, debugFile.owner);
+	const tmpPath = `${filePath}.${process.pid.toString()}.tmp`;
+	await writeFile(tmpPath, JSON.stringify(debugFile, null, 2));
+	await rename(tmpPath, filePath);
+}
 function pruneEntries(entries, namespace, maxEntries, protectedKey) {
 	const sorted = Object.values(entries).filter((entry) => entry.namespace === namespace).toSorted((a, b) => a.storedAt < b.storedAt ? 1 : -1);
 	const kept = /* @__PURE__ */ new Map();
@@ -4672,6 +5044,17 @@ function pruneEntries(entries, namespace, maxEntries, protectedKey) {
 	}
 	return Object.fromEntries(Object.values(entries).filter((entry) => entry.namespace !== namespace || kept.has(entry.key)).toSorted((a, b) => a.key < b.key ? -1 : 1).map((entry) => [entry.key, entry]));
 }
+function pruneDebugKeyEntries(entries, namespace, maxEntries, protectedKey) {
+	const sorted = Object.values(entries).filter((entry) => entry.namespace === namespace).toSorted((a, b) => a.storedAt < b.storedAt ? 1 : -1);
+	const kept = /* @__PURE__ */ new Map();
+	const protectedEntry = entries[protectedKey];
+	if (protectedEntry?.namespace === namespace) kept.set(protectedEntry.key, protectedEntry);
+	for (const entry of sorted) {
+		if (kept.size >= maxEntries) break;
+		kept.set(entry.key, entry);
+	}
+	return Object.fromEntries(Object.values(entries).filter((entry) => entry.namespace !== namespace || kept.has(entry.key)).toSorted((a, b) => a.key < b.key ? -1 : 1).map((entry) => [entry.key, entry]));
+}
 async function withCacheFileLock(filePath, fn) {
 	const lockPath = `${filePath}.lock`;
 	await acquireLock(lockPath);
@@ -4893,7 +5276,8 @@ const defaultConfig = {
 		label: "Output",
 		format: "json",
 		placements: ["section"]
-	}] }
+	}] },
+	runLogs: { captureConsole: true }
 };
 async function loadConfig() {
 	const configPath = resolve(process.cwd(), "agent-evals.config.ts");
@@ -5657,6 +6041,10 @@ async function runCase(params) {
 			} : void 0
 		});
 		const { trace, traceDisplay } = resolveTracePresentation(scoreRun.scope.spans, globalTraceDisplay, evalDef.traceDisplay);
+		scope.logs.push(...scoreRun.scope.logs.map((entry) => ({
+			...entry,
+			source: key
+		})));
 		if (trace.length > 0) scoringTraces[key] = {
 			trace,
 			traceDisplay
@@ -5727,6 +6115,7 @@ async function runCase(params) {
 		traceDisplay,
 		columns,
 		assertionFailures: scope.assertionFailures,
+		logs: scope.logs,
 		error: errorInfo,
 		trial,
 		cacheRefs: scope.caseCacheRefs
@@ -6142,4 +6531,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { caseRowSchema as $, appendToEvalOutput as $t, getEvalTitle as A, traceDisplayConfigSchema as At, apiCallMetricFormatSchema as B, fileRefSchema as Bt, createRunRequestSchema as C, serializedCacheSpanSchema as Ct, extractApiCalls as D, traceAttributeDisplayInputSchema as Dt, extractCacheHits as E, traceAttributeDisplayFormatSchema as Et, runManifestSchema as F, traceSpanWarningSchema as Ft, llmCallMetricPlacementSchema as G, z$1 as Gt, apiCallMetricSchema as H, numberDisplayOptionsSchema as Ht, runSummarySchema as I, cellValueSchema as It, resolveApiCallsConfig as J, evalSpan as Jt, llmCallMetricSchema as K, buildTraceTree as Kt, DEFAULT_API_CALLS_CONFIG as L, columnDefSchema as Lt, deriveScopedSummaryFromCases as M, traceSpanErrorSchema as Mt, deriveStatusFromCaseRows as N, traceSpanKindSchema as Nt, extractLlmCalls as O, traceAttributeDisplayPlacementSchema as Ot, deriveStatusFromChildStatuses as P, traceSpanSchema as Pt, caseDetailSchema as Q, EvalAssertionError as Qt, DEFAULT_LLM_CALLS_CONFIG as R, columnFormatSchema as Rt, createFsCacheStore as S, cacheStatusSchema as St, sseEnvelopeSchema as T, traceCacheRefSchema as Tt, apiCallsConfigSchema as U, repoFileRefSchema as Ut, apiCallMetricPlacementSchema as V, jsonCellSchema as Vt, llmCallMetricFormatSchema as W, runArtifactRefSchema as Wt, trialSelectionModeSchema as X, hashCacheKey as Xt, resolveLlmCallsConfig as Y, evalTracer as Yt, assertionFailureSchema as Z, hashCacheKeySync as Zt, loadEvalModule as _, cacheListItemSchema as _t, loadPersistedRunSnapshot as a, mergeEvalOutput as an, scoreTraceSchema as at, buildDeclaredColumnDefs as b, cacheRecordingOpSchema as bt, persistCaseDetail as c, runInEvalScope as cn, evalChartBuiltinMetricSchema as ct, recomputePersistedCaseStatus as d, setScopeCacheContext as dn, evalChartMetricSchema as dt, evalAssert as en, evalFreshnessStatusSchema as et, runTouchesEval as f, startEvalBackgroundJob as fn, evalChartTooltipExtraSchema as ft, setLatestRunInfoMap as g, cacheFileSchema as gt, getTargetEvalIds as h, getEvalRegistry as hn, cacheEntrySchema as ht, getLatestRunInfos as i, isInEvalScope as in, evalSummarySchema as it, getEvalDisplayStatus as j, traceDisplayInputConfigSchema as jt, getNestedAttribute as k, traceAttributeDisplaySchema as kt, persistRunState as l, runInExistingEvalScope as ln, evalChartColorSchema as lt, buildEvalSummary as m, defineEval as mn, evalChartsConfigSchema as mt, generateRunId as n, getEvalCaseInput as nn, evalStatItemSchema as nt, loadPersistedRunSnapshots as o, nextEvalId as on, evalChartAggregateSchema as ot, resolveArtifactPath as p, repoFile as pn, evalChartTypeSchema as pt, llmCallsConfigSchema as q, captureEvalSpanError as qt, getLastRunStatuses as r, incrementEvalOutput as rn, evalStatsConfigSchema as rt, nextShortIdFromSnapshots as s, runInEvalRuntimeScope as sn, evalChartAxisSchema as st, executeRun as t, getCurrentScope as tn, evalStatAggregateSchema as tt, recomputeEvalStatusesInRuns as u, setEvalOutput as un, evalChartConfigSchema as ut, parseEvalMetas as v, cacheModeSchema as vt, updateManualScoreRequestSchema as w, spanCacheOptionsSchema as wt, normalizeScoreDef as x, cacheRecordingSchema as xt, loadConfig as y, cacheOperationTypeSchema as yt, agentEvalsConfigSchema as z, columnKindSchema as zt };
+export { caseDetailSchema as $, buildTraceTree as $t, getEvalTitle as A, serializedCacheSpanSchema as At, apiCallMetricFormatSchema as B, traceSpanKindSchema as Bt, createRunRequestSchema as C, setEvalOutput as Cn, cacheFileSchema as Ct, extractApiCalls as D, defineEval as Dn, cacheRecordingOpSchema as Dt, extractCacheHits as E, repoFile as En, cacheOperationTypeSchema as Et, runManifestSchema as F, traceAttributeDisplayPlacementSchema as Ft, llmCallMetricPlacementSchema as G, columnFormatSchema as Gt, apiCallMetricSchema as H, traceSpanWarningSchema as Ht, runSummarySchema as I, traceAttributeDisplaySchema as It, resolveApiCallsConfig as J, jsonCellSchema as Jt, llmCallMetricSchema as K, columnKindSchema as Kt, DEFAULT_API_CALLS_CONFIG as L, traceDisplayConfigSchema as Lt, deriveScopedSummaryFromCases as M, traceCacheRefSchema as Mt, deriveStatusFromCaseRows as N, traceAttributeDisplayFormatSchema as Nt, extractLlmCalls as O, getEvalRegistry as On, cacheRecordingSchema as Ot, deriveStatusFromChildStatuses as P, traceAttributeDisplayInputSchema as Pt, assertionFailureSchema as Q, z$1 as Qt, DEFAULT_LLM_CALLS_CONFIG as R, traceDisplayInputConfigSchema as Rt, createFsCacheStore as S, runInExistingEvalScope as Sn, cacheEntryWithDebugKeySchema as St, sseEnvelopeSchema as T, startEvalBackgroundJob as Tn, cacheModeSchema as Tt, apiCallsConfigSchema as U, cellValueSchema as Ut, apiCallMetricPlacementSchema as V, traceSpanSchema as Vt, llmCallMetricFormatSchema as W, columnDefSchema as Wt, runLogsConfigSchema as X, repoFileRefSchema as Xt, resolveLlmCallsConfig as Y, numberDisplayOptionsSchema as Yt, trialSelectionModeSchema as Z, runArtifactRefSchema as Zt, loadEvalModule as _, isInEvalScope as _n, evalChartTypeSchema as _t, loadPersistedRunSnapshot as a, deserializeCacheRecording as an, evalSummarySchema as at, buildDeclaredColumnDefs as b, runInEvalRuntimeScope as bn, cacheDebugKeyFileSchema as bt, persistCaseDetail as c, serializeCacheValue as cn, runLogPhaseSchema as ct, recomputePersistedCaseStatus as d, configureEvalRunLogs as dn, evalChartAxisSchema as dt, captureEvalSpanError as en, caseRowSchema as et, runTouchesEval as f, evalAssert as fn, evalChartBuiltinMetricSchema as ft, setLatestRunInfoMap as g, incrementEvalOutput as gn, evalChartTooltipExtraSchema as gt, getTargetEvalIds as h, getEvalCaseInput as hn, evalChartMetricSchema as ht, getLatestRunInfos as i, hashCacheKeySync as in, evalStatsConfigSchema as it, getEvalDisplayStatus as j, spanCacheOptionsSchema as jt, getNestedAttribute as k, cacheStatusSchema as kt, persistRunState as l, EvalAssertionError as ln, scoreTraceSchema as lt, buildEvalSummary as m, getCurrentScope as mn, evalChartConfigSchema as mt, generateRunId as n, evalTracer as nn, evalStatAggregateSchema as nt, loadPersistedRunSnapshots as o, deserializeCacheValue as on, runLogEntrySchema as ot, resolveArtifactPath as p, evalLog as pn, evalChartColorSchema as pt, llmCallsConfigSchema as q, fileRefSchema as qt, getLastRunStatuses as r, hashCacheKey as rn, evalStatItemSchema as rt, nextShortIdFromSnapshots as s, serializeCacheRecording as sn, runLogLevelSchema as st, executeRun as t, evalSpan as tn, evalFreshnessStatusSchema as tt, recomputeEvalStatusesInRuns as u, appendToEvalOutput as un, evalChartAggregateSchema as ut, parseEvalMetas as v, mergeEvalOutput as vn, evalChartsConfigSchema as vt, updateManualScoreRequestSchema as w, setScopeCacheContext as wn, cacheListItemSchema as wt, normalizeScoreDef as x, runInEvalScope as xn, cacheEntrySchema as xt, loadConfig as y, nextEvalId as yn, cacheDebugKeyEntrySchema as yt, agentEvalsConfigSchema as z, traceSpanErrorSchema as zt };