@ls-stack/agent-eval 0.47.0 → 0.51.0

package/dist/{runOrchestration-BFdxG9ws.mjs → runOrchestration-o38J7uZO.mjs}

@@ -621,6 +621,8 @@ const hideIfNoValueShape = {
 hideIfNoValue: z.boolean().optional() };
 /**
 * One entry in the EvalCard stats row. Built-in kinds use latest run totals;
+* `cacheHits` counts Agent Eval operation-level cache hits from spans and
+* `evalTracer.cache(...)` refs, not LLM provider prompt-cache read tokens.
 * `column` aggregates a score or numeric output column across the latest run.
 */
 const evalStatItemSchema = z.discriminatedUnion("kind", [
@@ -637,6 +639,10 @@ const evalStatItemSchema = z.discriminatedUnion("kind", [
 		kind: z.literal("duration"),
 		...hideIfNoValueShape
 	}),
+	z.object({
+		kind: z.literal("cacheHits"),
+		...hideIfNoValueShape
+	}),
 	z.object({
 		kind: z.literal("column"),
 		key: z.string(),
@@ -731,6 +737,23 @@ const caseRowSchema = z.object({
 	]),
 	/** Elapsed case execution duration in milliseconds, or null before completion. */
 	durationMs: z.number().nullable(),
+	/**
+	* Agent Eval operation-level cache hits recorded for this case.
+	*
+	* This counts persisted operation cache hits from spans and
+	* `evalTracer.cache(...)` refs. It does not count LLM provider prompt-cache
+	* read tokens such as `cachedInputTokens`. Older run artifacts may omit it
+	* and should be treated as zero by aggregate readers.
+	*/
+	cacheHits: z.number().optional(),
+	/**
+	* Agent Eval operation-level cache activity entries recorded for this case.
+	*
+	* This is the denominator for `cacheHits`, counting hits plus misses and
+	* refreshes that appear in the Cache tab. Older run artifacts may omit it
+	* and should be treated as zero by aggregate readers.
+	*/
+	cacheOperations: z.number().optional(),
 	costUsd: z.number().nullable().optional(),
 	columns: z.record(z.string(), cellValueSchema),
 	/** Winning trial index for the persisted case result. */
@@ -771,7 +794,13 @@ const runLogLocationSchema = z.object({
 	/** 1-based source line reported by the JavaScript stack frame. */
 	line: z.number(),
 	/** 1-based source column reported by the JavaScript stack frame. */
-	column: z.number()
+	column: z.number(),
+	/**
+	* Full JavaScript stack captured when the log was emitted.
+	*
+	* Older run artifacts may only include the primary file, line, and column.
+	*/
+	stack: z.string().optional()
 });
 /** Schema for one persisted log entry captured during a case run. */
 const runLogEntrySchema = z.object({
@@ -1692,6 +1721,8 @@ function deriveScopedSummaryFromCases(params) {
 	let runningCases = 0;
 	let totalDurationMs = 0;
 	let hasDuration = false;
+	let cacheHits = 0;
+	let cacheOperations = 0;
 	for (const caseRow of caseRows) {
 		if (caseRow.status === "pass") passedCases += 1;
 		else if (caseRow.status === "fail") failedCases += 1;
@@ -1703,6 +1734,8 @@ function deriveScopedSummaryFromCases(params) {
 			totalDurationMs += caseRow.durationMs;
 			hasDuration = true;
 		}
+		cacheHits += caseRow.cacheHits ?? 0;
+		cacheOperations += caseRow.cacheOperations ?? 0;
 	}
 	return {
 		status: deriveStatusFromCaseRows({
@@ -1716,7 +1749,9 @@ function deriveScopedSummaryFromCases(params) {
 		cancelledCases,
 		pendingCases,
 		runningCases,
-		totalDurationMs: hasDuration ? totalDurationMs : null
+		totalDurationMs: hasDuration ? totalDurationMs : null,
+		cacheHits,
+		cacheOperations
 	};
 }
 //#endregion
@@ -2508,6 +2543,7 @@ function stripTerminalControlCodes$1(value) {
 const scopeStorage = new AsyncLocalStorage();
 const runtimeScopeStorage = new AsyncLocalStorage();
 const evalClockStorage = new AsyncLocalStorage();
+const activeSpanStackStorage = new AsyncLocalStorage();
 let activeEvalScopeCount = 0;
 let activeEvalRuntimeScopeCount = 0;
 let consoleCaptureEnabled = true;
@@ -2650,6 +2686,16 @@ function getCurrentScope() {
 	if (activeEvalScopeCount === 0) return void 0;
 	return scopeStorage.getStore();
 }
+/** Return the span currently active in this async execution, if any. */
+function getCurrentActiveSpan() {
+	if (activeEvalScopeCount === 0) return void 0;
+	return activeSpanStackStorage.getStore()?.at(-1);
+}
+/** Execute a callback with a span added to this async execution's active stack. */
+async function runWithActiveSpan(span, fn) {
+	const currentStack = activeSpanStackStorage.getStore() ?? [];
+	return await activeSpanStackStorage.run([...currentStack, span], fn);
+}
 /**
 * Return the current eval runner phase for this async execution.
 *
@@ -2787,7 +2833,8 @@ function normalizeStackFile(value) {
 	return decodeURIComponent(value.replace(fileUrlPrefixPattern, ""));
 }
 function isInternalLogFrame(file) {
-	return file.includes("/packages/sdk/src/runtime.ts") || file.includes("/node:internal/") || file.startsWith("node:internal/");
+	const normalizedFile = file.replaceAll("\\", "/");
+	return normalizedFile.includes("/packages/sdk/src/runtime.ts") || normalizedFile.includes("/packages/sdk/dist/") || normalizedFile.includes("/node_modules/@agent-evals/sdk/dist/") || normalizedFile.includes("/node_modules/@ls-stack/agent-eval/dist/") || normalizedFile.includes("/node:internal/") || normalizedFile.startsWith("node:internal/");
 }
 function parseStackFrameLocation(line) {
 	const match = stackFrameLocationPattern.exec(line.trim());
@@ -2808,7 +2855,10 @@ function getLogLocation() {
 	for (const line of stack.split("\n").slice(1)) {
 		const location = parseStackFrameLocation(line);
 		if (location === null || isInternalLogFrame(location.file)) continue;
-		return location;
+		return {
+			...location,
+			stack
+		};
 	}
 }
 function recordEvalLog(level, args) {
@@ -2951,8 +3001,6 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		logs: [],
 		spans: [],
 		checkpoints: /* @__PURE__ */ new Map(),
-		spanStack: [],
-		activeSpanStack: [],
 		recordingStack: [],
 		replayingDepth: 0,
 		cacheContext: options.cacheContext,
@@ -4185,7 +4233,7 @@ function createTraceCache(generateSpanId) {
 			namespace,
 			key: info.key
 		}, { serializeFileBytes: info.serializeFileBytes === true });
-		const activeSpan = scope.activeSpanStack.at(-1);
+		const activeSpan = getCurrentActiveSpan();
 		const canRead = cacheCtx.mode === "use" && cacheCtx.read !== false;
 		const canStore = cacheCtx.mode !== "bypass" && cacheCtx.store !== false;
 		if (canRead) {
@@ -4283,7 +4331,7 @@ function generateSpanId() {
 	return `span_${String(Date.now())}_${String(spanIdCounter)}`;
 }
 function updateCurrentSpan(update) {
-	const currentSpan = getCurrentScope()?.activeSpanStack.at(-1);
+	const currentSpan = getCurrentActiveSpan();
 	if (!currentSpan) return;
 	update(currentSpan);
 }
@@ -4454,9 +4502,9 @@ function toIsoTimestamp(value) {
 function findSpan(scope, id) {
 	return scope.spans.find((span) => span.id === id);
 }
-function resolveExternalParentId(scope, parentId) {
+function resolveExternalParentId(parentId) {
 	if (parentId !== void 0) return parentId;
-	return scope.activeSpanStack.at(-1)?.id ?? null;
+	return getCurrentActiveSpan()?.id ?? null;
 }
 function startExternalSpan(info) {
 	const id = info.id ?? generateSpanId();
@@ -4464,7 +4512,7 @@ function startExternalSpan(info) {
 	if (!scope) return noopExternalSpan(id);
 	const existing = findSpan(scope, id);
 	if (existing) {
-		existing.parentId = resolveExternalParentId(scope, info.parentId);
+		existing.parentId = resolveExternalParentId(info.parentId);
 		existing.kind = info.kind;
 		existing.name = info.name;
 		existing.startedAt = toIsoTimestamp(info.startedAt);
@@ -4475,7 +4523,7 @@ function startExternalSpan(info) {
 	}
 	scope.spans.push({
 		id,
-		parentId: resolveExternalParentId(scope, info.parentId),
+		parentId: resolveExternalParentId(info.parentId),
 		caseId: scope.caseId,
 		kind: info.kind,
 		name: info.name,
@@ -4516,7 +4564,7 @@ function recordExternalSpan(info) {
 	const existing = findSpan(scope, id);
 	const status = info.status ?? (info.error ? "error" : "ok");
 	if (existing) {
-		existing.parentId = resolveExternalParentId(scope, info.parentId);
+		existing.parentId = resolveExternalParentId(info.parentId);
 		existing.kind = info.kind;
 		existing.name = info.name;
 		existing.startedAt = startedAt;
@@ -4530,7 +4578,7 @@ function recordExternalSpan(info) {
 	}
 	scope.spans.push({
 		id,
-		parentId: resolveExternalParentId(scope, info.parentId),
+		parentId: resolveExternalParentId(info.parentId),
 		caseId: scope.caseId,
 		kind: info.kind,
 		name: info.name,
@@ -4613,7 +4661,7 @@ async function traceSpanInternal(info, fn) {
 	const scope = getCurrentScope();
 	if (!scope) return await fn(noopActiveSpan());
 	const id = generateSpanId();
-	const parentId = scope.activeSpanStack.at(-1)?.id ?? null;
+	const parentId = getCurrentActiveSpan()?.id ?? null;
 	const realStartedAt = getRealDateNowMs();
 	const spanRecord = {
 		id,
@@ -4627,109 +4675,106 @@ async function traceSpanInternal(info, fn) {
 		attributes: info.attributes
 	};
 	scope.spans.push(spanRecord);
-	scope.spanStack.push(id);
-	scope.activeSpanStack.push(spanRecord);
 	const activeSpan = createSpanHandle(spanRecord);
-	try {
-		const cacheOpts = info.cache;
-		const cacheCtx = scope.cacheContext;
-		if (cacheOpts !== void 0 && cacheCtx !== void 0 && scope.replayingDepth === 0) {
-			const ctx = cacheCtx;
-			const namespace = getRequiredSpanCacheNamespace(cacheOpts);
-			const keyHash = await hashCacheKey({
-				namespace,
-				key: cacheOpts.key
-			}, { serializeFileBytes: cacheOpts.serializeFileBytes === true });
-			const canRead = ctx.mode === "use" && ctx.read !== false;
-			const canStore = ctx.mode !== "bypass" && ctx.store !== false;
-			mergeSpanAttributes(spanRecord, {
-				"cache.key": keyHash,
-				"cache.namespace": namespace
-			});
-			if (canRead) {
-				const hit = await ctx.adapter.lookup(namespace, keyHash);
-				if (hit) {
-					const storedAt = hit.storedAt;
+	return await runWithActiveSpan(spanRecord, async () => {
+		try {
+			const cacheOpts = info.cache;
+			const cacheCtx = scope.cacheContext;
+			if (cacheOpts !== void 0 && cacheCtx !== void 0 && scope.replayingDepth === 0) {
+				const ctx = cacheCtx;
+				const namespace = getRequiredSpanCacheNamespace(cacheOpts);
+				const keyHash = await hashCacheKey({
+					namespace,
+					key: cacheOpts.key
+				}, { serializeFileBytes: cacheOpts.serializeFileBytes === true });
+				const canRead = ctx.mode === "use" && ctx.read !== false;
+				const canStore = ctx.mode !== "bypass" && ctx.store !== false;
+				mergeSpanAttributes(spanRecord, {
+					"cache.key": keyHash,
+					"cache.namespace": namespace
+				});
+				if (canRead) {
+					const hit = await ctx.adapter.lookup(namespace, keyHash);
+					if (hit) {
+						const storedAt = hit.storedAt;
+						mergeSpanAttributes(spanRecord, {
+							"cache.status": "hit",
+							"cache.storedAt": storedAt,
+							"cache.age": getRealDateNowMs() - new Date(storedAt).getTime()
+						});
+						const recording = deserializeCacheRecording(hit.recording);
+						replayRecording(scope, spanRecord, recording, { generateSpanId });
+						spanRecord.status = recording.finalStatus ?? (hasSpanError(spanRecord) ? "error" : "ok");
+						spanRecord.endedAt = addElapsedMsToTimestamp(spanRecord.startedAt, getRealDateNowMs() - realStartedAt);
+						return recording.returnValue;
+					}
 					mergeSpanAttributes(spanRecord, {
-						"cache.status": "hit",
-						"cache.storedAt": storedAt,
-						"cache.age": getRealDateNowMs() - new Date(storedAt).getTime()
+						"cache.status": "miss",
+						...canStore ? {} : { "cache.stored": false }
 					});
-					const recording = deserializeCacheRecording(hit.recording);
-					replayRecording(scope, spanRecord, recording, { generateSpanId });
-					spanRecord.status = recording.finalStatus ?? (hasSpanError(spanRecord) ? "error" : "ok");
-					spanRecord.endedAt = addElapsedMsToTimestamp(spanRecord.startedAt, getRealDateNowMs() - realStartedAt);
-					return recording.returnValue;
-				}
-				mergeSpanAttributes(spanRecord, {
+				} else if (ctx.mode === "use" && canStore) mergeSpanAttributes(spanRecord, {
 					"cache.status": "miss",
+					"cache.read": false
+				});
+				else if (ctx.mode === "refresh") mergeSpanAttributes(spanRecord, {
+					"cache.status": "refresh",
 					...canStore ? {} : { "cache.stored": false }
 				});
-			} else if (ctx.mode === "use" && canStore) mergeSpanAttributes(spanRecord, {
-				"cache.status": "miss",
-				"cache.read": false
-			});
-			else if (ctx.mode === "refresh") mergeSpanAttributes(spanRecord, {
-				"cache.status": "refresh",
-				...canStore ? {} : { "cache.stored": false }
-			});
-			else mergeSpanAttributes(spanRecord, { "cache.status": "bypass" });
-			const frame = {
-				baseSpanIndex: scope.spans.length,
-				replayParentSpanId: id,
-				ops: []
-			};
-			scope.recordingStack.push(frame);
-			let bodyResult;
-			try {
-				bodyResult = await fn(activeSpan);
-			} finally {
-				scope.recordingStack.pop();
-			}
-			appendSubSpanOps(scope, frame);
-			finishSpanWithoutThrownError(spanRecord, realStartedAt);
-			if (canStore) {
-				const recording = {
-					returnValue: bodyResult,
-					finalAttributes: stripCacheAttributes(spanRecord.attributes),
-					finalStatus: spanRecord.status,
-					finalError: spanRecord.error,
-					finalErrors: spanRecord.errors,
-					finalWarning: spanRecord.warning,
-					finalWarnings: spanRecord.warnings,
-					ops: frame.ops
-				};
-				const entry = {
-					version: 1,
-					key: keyHash,
-					namespace,
-					operationType: "span",
-					operationName: info.name,
-					spanName: info.name,
-					spanKind: info.kind,
-					storedAt: new Date(getRealDateNowMs()).toISOString(),
-					recording: await serializeCacheRecording(recording, { externalJsonStore: ctx.adapter.externalJsonStore })
+				else mergeSpanAttributes(spanRecord, { "cache.status": "bypass" });
+				const frame = {
+					baseSpanIndex: scope.spans.length,
+					replayParentSpanId: id,
+					ops: []
 				};
-				await ctx.adapter.write(entry, {
-					rawKey: cacheOpts.key,
-					operationType: "span",
-					operationName: info.name
-				});
+				scope.recordingStack.push(frame);
+				let bodyResult;
+				try {
+					bodyResult = await fn(activeSpan);
+				} finally {
+					scope.recordingStack.pop();
+				}
+				appendSubSpanOps(scope, frame);
+				finishSpanWithoutThrownError(spanRecord, realStartedAt);
+				if (canStore) {
+					const recording = {
+						returnValue: bodyResult,
+						finalAttributes: stripCacheAttributes(spanRecord.attributes),
+						finalStatus: spanRecord.status,
+						finalError: spanRecord.error,
+						finalErrors: spanRecord.errors,
+						finalWarning: spanRecord.warning,
+						finalWarnings: spanRecord.warnings,
+						ops: frame.ops
+					};
+					const entry = {
+						version: 1,
+						key: keyHash,
+						namespace,
+						operationType: "span",
+						operationName: info.name,
+						spanName: info.name,
+						spanKind: info.kind,
+						storedAt: new Date(getRealDateNowMs()).toISOString(),
+						recording: await serializeCacheRecording(recording, { externalJsonStore: ctx.adapter.externalJsonStore })
+					};
+					await ctx.adapter.write(entry, {
+						rawKey: cacheOpts.key,
+						operationType: "span",
+						operationName: info.name
+					});
+				}
+				return bodyResult;
 			}
-			return bodyResult;
+			const result = await fn(activeSpan);
+			finishSpanWithoutThrownError(spanRecord, realStartedAt);
+			return result;
+		} catch (error) {
+			spanRecord.status = "error";
+			spanRecord.endedAt = addElapsedMsToTimestamp(spanRecord.startedAt, getRealDateNowMs() - realStartedAt);
+			spanRecord.error = normalizeTraceError(error);
+			throw error;
 		}
-		const result = await fn(activeSpan);
-		finishSpanWithoutThrownError(spanRecord, realStartedAt);
-		return result;
-	} catch (error) {
-		spanRecord.status = "error";
-		spanRecord.endedAt = addElapsedMsToTimestamp(spanRecord.startedAt, getRealDateNowMs() - realStartedAt);
-		spanRecord.error = normalizeTraceError(error);
-		throw error;
-	} finally {
-		scope.spanStack.pop();
-		scope.activeSpanStack.pop();
-	}
+	});
 }
 function getRequiredSpanCacheNamespace(cacheOpts) {
 	if (!isRecordLike$1(cacheOpts)) throw new Error("Cached spans require a non-empty cache.namespace");
@@ -4785,7 +4830,7 @@ const evalTracer = {
 		if (!scope) return;
 		scope.checkpoints.set(name, data);
 		const id = generateSpanId();
-		const parentId = scope.spanStack.at(-1) ?? null;
+		const parentId = getCurrentActiveSpan()?.id ?? null;
 		scope.spans.push({
 			id,
 			parentId,
@@ -7169,12 +7214,16 @@ async function runCase(params) {
 	};
 	if (Object.keys(scoringTraces).length > 0) caseDetail.scoringTraces = scoringTraces;
 	const elapsedMs = Date.now() - startTime;
+	const cacheEntries = extractCacheEntries(displayTrace, scope.caseCacheRefs);
+	const cacheHits = cacheEntries.filter((entry) => entry.status === "hit");
 	return {
 		caseDetail,
 		caseRowUpdate: {
 			tags: evalCase.tags ?? [],
 			status,
 			durationMs: elapsedMs,
+			cacheHits: cacheHits.length,
+			cacheOperations: cacheEntries.length,
 			columns
 		}
 	};
@@ -7670,6 +7719,8 @@ async function executeRun({ runState, request, runDir, config, cacheStore, lastR
 													tags: caseDetail.tags,
 													status: caseRowUpdate.status ?? "pending",
 													durationMs: caseRowUpdate.durationMs ?? null,
+													cacheHits: caseRowUpdate.cacheHits ?? 0,
+													cacheOperations: caseRowUpdate.cacheOperations ?? 0,
 													columns: caseRowUpdate.columns ?? {},
 													trial
 												}

package/dist/{runner-DJWn_7p0.mjs → runner-LdMiDmAN.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-vdJYkEVk.mjs";
-import "./src-BRqs3kSA.mjs";
+import { n as createRunner } from "./cli-Cvs7tc2v.mjs";
+import "./src-Jahivm6d.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{runner--aH0jO4Z.mjs → runner-iWtmKx9z.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-DJWn_7p0.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-LdMiDmAN.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{src-BRqs3kSA.mjs → src-Jahivm6d.mjs}

@@ -1,5 +1,5 @@
-import { It as defineEval$1, rt as matchesEvalTags$1 } from "./runOrchestration-BFdxG9ws.mjs";
-import "./cli-vdJYkEVk.mjs";
+import { It as defineEval$1, rt as matchesEvalTags$1 } from "./runOrchestration-o38J7uZO.mjs";
+import "./cli-Cvs7tc2v.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.47.0",
+  "version": "0.51.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -33,8 +33,8 @@
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/sdk": "0.0.1",
-    "@agent-evals/shared": "0.0.1"
+    "@agent-evals/shared": "0.0.1",
+    "@agent-evals/sdk": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",

package/skills/agent-eval/SKILL.md

@@ -400,11 +400,17 @@ cacheCreationInputTokens` so cache details are not double-counted.
 - `runLogs` (in `agent-evals.config.ts`) controls case log capture. Use
   `runLogs: { captureConsole: false }` to keep console output in the terminal
   without persisting console calls to case details. Manual `evalLog(...)` calls
-  are still captured.
+  are still captured. Captured log locations store the selected user-facing
+  source frame and the full JavaScript stack so agents can inspect additional
+  frames in persisted artifacts when diagnosing where a log came from.
 
 Stats rows and history charts can be authored via `stats` / `charts` on the eval
 definition. Global `stats` in `agent-evals.config.ts` combine with eval-level
-stats. Usage stats and LLM usage charts are added by default unless removed with
+stats. Native stat kinds include `cases`, `passRate`, `duration`, and
+`cacheHits`; `cacheHits` shows Agent Eval operation-level cache hits over total
+cache operations (`hits/total`) from spans and `evalTracer.cache(...)` refs, not
+LLM provider prompt-cache read tokens such as `cachedInputTokens`. Usage stats
+and LLM usage charts are added by default unless removed with
 `removeDefaultConfig`. Column stats can override `format` and `numberFormat`,
 otherwise they inherit from the matching column. Number formats use
 `maxDecimalPlaces` to cap decimals and `minDecimalPlaces` to pad trailing