@ls-stack/agent-eval 0.57.0 → 0.58.0

package/dist/{runExecution-BH7DlMXl.mjs → runExecution-C4kAOhC1.mjs}

@@ -289,7 +289,8 @@ z.object({
 	key: z.string(),
 	namespace: z.string(),
 	storedAt: z.string(),
-	lastAccessedAt: z.string()
+	/** Last successful cache hit time. `null` means the entry has not been hit yet. */
+	lastAccessedAt: z.string().nullable()
 });
 z.object({
 	removedCacheFiles: z.number(),
@@ -814,8 +815,7 @@ const caseRowSchema = z.object({
 	/** Winning trial index for the persisted case result. */
 	trial: z.number()
 });
-/** Structured assertion failure metadata captured for one case run. */
-const assertionFailureSchema = z.object({
+const assertionBaseSchema = z.object({
 	/**
 	* Error class or category label rendered alongside the message (e.g.
 	* `EvalAssertionError`, `OutputsSchemaError`). Optional for legacy entries
@@ -827,7 +827,19 @@ const assertionFailureSchema = z.object({
 	/** Stack trace captured from the originating error when available. */
 	stack: z.string().optional()
 });
+/** Structured assertion failure metadata captured for one case run. */
+const assertionFailureSchema = assertionBaseSchema;
+/** Pass/fail outcome for one recorded eval assertion. */
+const assertionStatusSchema = z.enum(["pass", "fail"]);
+/** Structured assertion result metadata captured for one case run. */
+const assertionResultSchema = assertionBaseSchema.extend({ 
+/** Whether the recorded assertion passed or failed. */
+status: assertionStatusSchema });
 const legacyAssertionFailureSchema = z.string().transform((message) => ({ message }));
+const legacyAssertionResultSchema = z.string().transform((message) => ({
+	message,
+	status: "fail"
+}));
 /** Severity level for one log captured during a case run. */
 const runLogLevelSchema = z.enum([
 	"log",
@@ -922,6 +934,12 @@ const caseDetailSchema = z.object({
 	* These complement eval-level `columns` without changing discovery metadata.
 	*/
 	outputColumnDefs: z.array(columnDefSchema).optional(),
+	/**
+	* Pass/fail assertion records captured from eval assertion helpers. New run
+	* artifacts include this alongside `assertionFailures`; older artifacts may
+	* omit it and should fall back to `assertionFailures` for failed outcomes.
+	*/
+	assertions: z.array(z.union([assertionResultSchema, legacyAssertionResultSchema])).optional(),
 	assertionFailures: z.array(z.union([assertionFailureSchema, legacyAssertionFailureSchema])),
 	/** Logs captured from manual `evalLog(...)` calls and enabled console calls. */
 	logs: z.array(runLogEntrySchema).default([]),
@@ -1405,6 +1423,7 @@ const agentEvalsConfigSchema = z.object({
 		maxEntriesPerNamespace: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
 		maxEntriesByNamespace: z.record(z.string(), z.number()).optional(),
 		pruneIdleDelayMs: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
+		lastAccessedAtUpdateIntervalMs: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
 		maxEntriesPerEval: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional())
 	}).optional()
 });
@@ -3086,6 +3105,7 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		tags: options.tags ?? [],
 		outputs: {},
 		outputColumnOverrides: {},
+		assertions: [],
 		assertionFailures: [],
 		logs: [],
 		spans: [],
@@ -3211,7 +3231,7 @@ function mergeEvalOutput(key, patch) {
 		return;
 	}
 	if (!isObjectRecord(existing)) {
-		scope.assertionFailures.push(toAssertionFailure$1(`mergeEvalOutput("${key}"): existing value is ${Array.isArray(existing) ? "array" : typeof existing}, expected object`));
+		recordAssertionFailure$1(scope, toAssertionFailure$1(`mergeEvalOutput("${key}"): existing value is ${Array.isArray(existing) ? "array" : typeof existing}, expected object`));
 		return;
 	}
 	scope.outputs[key] = {
@@ -3244,7 +3264,7 @@ function incrementEvalOutput(key, delta) {
 		return;
 	}
 	if (typeof existing !== "number") {
-		scope.assertionFailures.push(toAssertionFailure$1(`incrementEvalOutput("${key}"): existing value is ${typeof existing}, expected number`));
+		recordAssertionFailure$1(scope, toAssertionFailure$1(`incrementEvalOutput("${key}"): existing value is ${typeof existing}, expected number`));
 		return;
 	}
 	scope.outputs[key] = existing + delta;
@@ -3263,13 +3283,26 @@ function incrementEvalOutput(key, delta) {
 * call.
 */
 function evalAssert(condition, message) {
-	if (condition) return;
 	const scope = getCurrentScope();
+	if (condition) {
+		if (scope) scope.assertions.push({
+			message,
+			status: "pass"
+		});
+		return;
+	}
 	if (!scope) return;
 	const error = new EvalAssertionError(message);
-	scope.assertionFailures.push(toAssertionFailure$1(message, error));
+	recordAssertionFailure$1(scope, toAssertionFailure$1(message, error));
 	throw error;
 }
+function recordAssertionFailure$1(scope, failure) {
+	scope.assertionFailures.push(failure);
+	scope.assertions.push({
+		...failure,
+		status: "fail"
+	});
+}
 //#endregion
 //#region ../sdk/src/evalExpect.ts
 const expectFormatOptions = {
@@ -4185,14 +4218,28 @@ function applyRecordingOp(scope, parentSpan, op, options) {
 			...existing,
 			...op.patch
 		};
-		else scope.assertionFailures.push({ message: `replay mergeEvalOutput("${op.key}"): existing value is ${valueKind$1(existing)}, expected object` });
+		else {
+			const message = `replay mergeEvalOutput("${op.key}"): existing value is ${valueKind$1(existing)}, expected object`;
+			scope.assertionFailures.push({ message });
+			scope.assertions.push({
+				message,
+				status: "fail"
+			});
+		}
 		return;
 	}
 	if (op.kind === "incrementOutput") {
 		const existing = scope.outputs[op.key];
 		if (existing === void 0) scope.outputs[op.key] = op.delta;
 		else if (typeof existing === "number") scope.outputs[op.key] = existing + op.delta;
-		else scope.assertionFailures.push({ message: `replay incrementEvalOutput("${op.key}"): existing value is ${valueKind$1(existing)}, expected number` });
+		else {
+			const message = `replay incrementEvalOutput("${op.key}"): existing value is ${valueKind$1(existing)}, expected number`;
+			scope.assertionFailures.push({ message });
+			scope.assertions.push({
+				message,
+				status: "fail"
+			});
+		}
 		return;
 	}
 	if (op.kind === "checkpoint") {
@@ -4478,6 +4525,10 @@ function recordSpanAttributeAssertion(message) {
 	const scope = getCurrentScope();
 	if (!scope) return;
 	scope.assertionFailures.push({ message });
+	scope.assertions.push({
+		message,
+		status: "fail"
+	});
 }
 function incrementSpanAttribute(span, key, delta) {
 	const existing = span.attributes?.[key];
@@ -4983,6 +5034,24 @@ function buildTraceTree(spans, checkpoints) {
 	};
 }
 //#endregion
+//#region ../runner/src/cacheAccessTime.ts
+const defaultLastAccessedAtUpdateIntervalMs = 14400 * 1e3;
+function normalizeLastAccessedAtUpdateIntervalMs(value) {
+	if (value === void 0 || !Number.isFinite(value) || value < 0) return defaultLastAccessedAtUpdateIntervalMs;
+	return Math.floor(value);
+}
+function cacheAccessSortTime(entry) {
+	return entry.lastAccessedAt ?? entry.storedAt;
+}
+function shouldRefreshLastAccessedAt(params) {
+	return params.lastAccessedAt === null || params.nowMs - Date.parse(params.lastAccessedAt) > params.updateIntervalMs;
+}
+//#endregion
+//#region ../runner/src/cacheKeys.ts
+function toPendingKey(namespace, keyHash) {
+	return `${namespace}::${keyHash}`;
+}
+//#endregion
 //#region ../runner/src/cacheStore.ts
 const defaultMaxEntriesPerNamespace = 100;
 const cacheSerializationMarker = "__aecs";
@@ -5015,6 +5084,7 @@ function createFsCacheStore(options) {
 		primaryDir: blobDir
 	});
 	const defaultMaxEntries = normalizeMaxEntries(options.maxEntriesPerNamespace);
+	const lastAccessedAtUpdateIntervalMs = normalizeLastAccessedAtUpdateIntervalMs(options.lastAccessedAtUpdateIntervalMs);
 	return {
 		externalJsonStore,
 		dir() {
@@ -5034,7 +5104,12 @@ function createFsCacheStore(options) {
 			});
 			if (entry === null) return null;
 			const materialized = await materializeExternalJsonCacheEntryOrNull(entry, externalJsonStore);
-			if (materialized !== null) await updateCacheIndexLastAccessedAt(cacheDir, namespace, keyHash);
+			if (materialized !== null) await updateCacheIndexLastAccessedAt({
+				cacheDir,
+				key: keyHash,
+				namespace,
+				updateIntervalMs: lastAccessedAtUpdateIntervalMs
+			});
 			return materialized;
 		},
 		async lookupWithDebug(namespace, keyHash) {
@@ -5063,7 +5138,7 @@ function createFsCacheStore(options) {
 				const index = await readNamespaceIndex(cacheDir, entry.namespace);
 				index.entries[entry.key] = {
 					storedAt: entry.storedAt,
-					lastAccessedAt: entry.storedAt,
+					lastAccessedAt: null,
 					blobRefs: await collectExternalJsonBlobRefs(entry, blobDirs)
 				};
 				await writeNamespaceIndex(cacheDir, index);
@@ -5082,7 +5157,7 @@ function createFsCacheStore(options) {
 		async list() {
 			const items = [];
 			for (const index of await listCacheIndexes(cacheDir)) for (const [key, entry] of Object.entries(index.entries)) items.push(toCacheListItem(index.namespace, key, entry));
-			items.sort((a, b) => a.lastAccessedAt < b.lastAccessedAt ? 1 : -1);
+			items.sort((a, b) => cacheAccessSortTime(a) < cacheAccessSortTime(b) ? 1 : -1);
 			return items;
 		},
 		async clear(filter) {
@@ -5183,9 +5258,6 @@ function maxEntriesForNamespace(namespace, defaultMaxEntries, maxEntriesByNamesp
 	const namespaceMaxEntries = maxEntriesByNamespace?.[namespace];
 	return namespaceMaxEntries === void 0 ? defaultMaxEntries : normalizeMaxEntries(namespaceMaxEntries, defaultMaxEntries);
 }
-function toPendingKey(namespace, keyHash) {
-	return `${namespace}::${keyHash}`;
-}
 function sanitizeSegment$1(segment) {
 	return segment.replace(/[^a-zA-Z0-9_.-]/g, "_");
 }
@@ -5231,16 +5303,22 @@ async function readIndexedCacheEntry(params) {
 		return fileEntry.entry;
 	});
 }
-async function updateCacheIndexLastAccessedAt(cacheDir, namespace, key) {
-	await withCacheFileLock(namespaceLockPath(cacheDir, namespace), async () => {
-		const index = await readNamespaceIndex(cacheDir, namespace);
-		const entry = index.entries[key];
+async function updateCacheIndexLastAccessedAt(params) {
+	await withCacheFileLock(namespaceLockPath(params.cacheDir, params.namespace), async () => {
+		const index = await readNamespaceIndex(params.cacheDir, params.namespace);
+		const entry = index.entries[params.key];
 		if (entry === void 0) return;
-		index.entries[key] = {
+		const nowMs = getRealDateNowMs();
+		if (!shouldRefreshLastAccessedAt({
+			lastAccessedAt: entry.lastAccessedAt,
+			nowMs,
+			updateIntervalMs: params.updateIntervalMs
+		})) return;
+		index.entries[params.key] = {
 			...entry,
-			lastAccessedAt: new Date(getRealDateNowMs()).toISOString()
+			lastAccessedAt: new Date(nowMs).toISOString()
 		};
-		await writeNamespaceIndex(cacheDir, index);
+		await writeNamespaceIndex(params.cacheDir, index);
 	});
 }
 async function readCacheEntryFilePath(filePath, expected) {
@@ -5371,7 +5449,7 @@ function parseCacheIndexFile(value, expectedNamespace) {
 }
 function parseCacheIndexEntry(value) {
 	if (!isRecordLike(value)) return null;
-	if (typeof value.storedAt !== "string" || typeof value.lastAccessedAt !== "string") return null;
+	if (typeof value.storedAt !== "string" || value.lastAccessedAt !== null && typeof value.lastAccessedAt !== "string") return null;
 	if (!Array.isArray(value.blobRefs)) return null;
 	const blobRefs = [];
 	for (const blobRef of value.blobRefs) {
@@ -5441,7 +5519,7 @@ function entryMatchesFilter(entry, filter) {
 async function pruneCacheEntriesForNamespace(params) {
 	const { cacheDir, index, maxEntries } = params;
 	const entries = Object.entries(index.entries);
-	const sorted = entries.toSorted(([, a], [, b]) => a.lastAccessedAt < b.lastAccessedAt ? 1 : -1);
+	const sorted = entries.toSorted(([, a], [, b]) => cacheAccessSortTime(a) < cacheAccessSortTime(b) ? 1 : -1);
 	const keptKeys = /* @__PURE__ */ new Set();
 	for (const [key] of sorted) {
 		if (keptKeys.size >= maxEntries) break;
@@ -6606,7 +6684,7 @@ async function runDeriveFromTracingConfig(params) {
 		});
 	} catch (e) {
 		const message = `deriveFromTracing threw: ${e instanceof Error ? e.message : String(e)}`;
-		params.scope.assertionFailures.push(toAssertionFailure(message, e instanceof Error ? e : void 0));
+		recordAssertionFailure(params.scope, toAssertionFailure(message, e instanceof Error ? e : void 0));
 	}
 }
 async function runCase(params) {
@@ -6656,7 +6734,7 @@ async function runCase(params) {
 	});
 	const traceTree = buildTraceTree(spansWithDerivedAttributes, scope.checkpoints);
 	const nonAssertError = executeError && !(executeError instanceof EvalAssertionError) ? executeError : null;
-	if (executeError instanceof EvalAssertionError && scope.assertionFailures.length === 0) scope.assertionFailures.push(toAssertionFailure(executeError.message, executeError));
+	if (executeError instanceof EvalAssertionError && scope.assertionFailures.length === 0) recordAssertionFailure(scope, toAssertionFailure(executeError.message, executeError));
 	if (!nonAssertError) {
 		await runDeriveFromTracingConfig({
 			deriveFromTracing: globalDeriveFromTracing,
@@ -6686,7 +6764,7 @@ async function runCase(params) {
 			...scope.outputs,
 			...parsedOutputs.data
 		};
-		else scope.assertionFailures.push(toAssertionFailure(formatOutputsSchemaError(parsedOutputs.error), void 0, "OutputsSchemaError"));
+		else recordAssertionFailure(scope, toAssertionFailure(formatOutputsSchemaError(parsedOutputs.error), void 0, "OutputsSchemaError"));
 	}
 	const scoreResults = /* @__PURE__ */ new Map();
 	const scoringTraces = {};
@@ -6728,8 +6806,7 @@ async function runCase(params) {
 		};
 		const rawValue = scoreRun.result;
 		if (scoreRun.error) {
-			const message = `score "${key}" threw: ${scoreRun.error.message}`;
-			scope.assertionFailures.push(toAssertionFailure(message, scoreRun.error));
+			recordAssertionFailure(scope, toAssertionFailure(`score "${key}" threw: ${scoreRun.error.message}`, scoreRun.error));
 			scope.outputs[key] = 0;
 			scoreResults.set(key, {
 				value: 0,
@@ -6739,7 +6816,7 @@ async function runCase(params) {
 			continue;
 		}
 		if (typeof rawValue !== "number") {
-			scope.assertionFailures.push(toAssertionFailure(`score "${key}" must return a number`));
+			recordAssertionFailure(scope, toAssertionFailure(`score "${key}" must return a number`));
 			scope.outputs[key] = 0;
 			scoreResults.set(key, {
 				value: 0,
@@ -6801,6 +6878,7 @@ async function runCase(params) {
 		traceDisplay,
 		columns,
 		...outputColumnDefs.length > 0 ? { outputColumnDefs } : {},
+		assertions: scope.assertions,
 		assertionFailures: scope.assertionFailures,
 		logs: scope.logs,
 		error: errorInfo,
@@ -6852,5 +6930,12 @@ function toAssertionFailure(message, error = void 0, nameOverride = void 0) {
 		...stack !== void 0 ? { stack } : {}
 	};
 }
+function recordAssertionFailure(scope, failure) {
+	scope.assertionFailures.push(failure);
+	scope.assertions.push({
+		...failure,
+		status: "fail"
+	});
+}
 //#endregion
 export { startEvalBackgroundJob as $, repoFile as A, evalChartsConfigSchema as At, getCurrentScope as B, evalTracer as C, buildEvalKey as Ct, deserializeCacheValue as D, evalStatAggregateSchema as Dt, deserializeCacheRecording as E, caseRowSchema as Et, appendToEvalOutput as F, mergeEvalOutput as G, incrementEvalOutput as H, configureEvalRunLogs as I, runInEvalScope as J, nextEvalId as K, evalAssert as L, readManualInputFile as M, defineEval as Mt, evalExpect as N, getEvalRegistry as Nt, serializeCacheRecording as O, evalStatsConfigSchema as Ot, EvalAssertionError as P, runWithEvalRegistry as Pt, setScopeCacheContext as Q, evalLog as R, evalSpan as S, resolveLlmCallsConfig as St, hashCacheKeySync as T, caseDetailSchema as Tt, isInEvalScope as U, getEvalCaseInput as V, matchesEvalTags as W, runWithEvalClock as X, runInExistingEvalScope as Y, setEvalOutput as Z, createBufferedCacheStore as _, validateEvalTagName as _t, isCaseChildParentMessage as a, extractLlmCalls as at, buildTraceTree as b, runSummarySchema as bt, resolveArtifactPath as c, applyDerivedCallAttributes as ct, loadEvalModule as d, getEvalDisplayStatus as dt, createRunRequestSchema as et, resolveEvalDefaultConfig as f, deriveScopedSummaryFromCases as ft, commitPendingCacheWrites as g, matchesTagsFilter as gt, normalizeScoreDef as h, dedupeEvalTags as ht, isCaseChildMessage as i, extractApiCalls as it, manualInputFileValueSchema as j, columnDefSchema as jt, serializeCacheValue as k, manualInputDescriptorSchema as kt, registerAgentEvalsPackageResolutionHooks as l, getNestedAttribute as lt, buildDeclaredColumnDefs as m, deriveStatusFromChildStatuses as mt, resolveRunnableEvalCases as n, extractCacheEntries as nt, stripTerminalControlCodes as o, simulateLlmCallCost as ot, loadConfig as p, deriveStatusFromCaseRows as pt, runInEvalRuntimeScope as q, runCase as r, extractCacheHits as rt, resolveTracePresentation as s, simulateTokenAllocation as st, filterEvalCases as t, updateManualScoreRequestSchema as tt, runWithModuleIsolation as u, getEvalTitle as ut, createFsCacheStore as v, validateTagsFilterExpression as vt, hashCacheKey as w, getCaseRowCaseKey as wt, captureEvalSpanError as x, resolveApiCallsConfig as xt, z$1 as y, runManifestSchema as yt, evalTime as z };

package/dist/{runOrchestration-C1Ex9QI-.mjs → runOrchestration-5xEiQxiS.mjs}

@@ -1,4 +1,4 @@
-import { Et as caseRowSchema, Pt as runWithEvalRegistry, Tt as caseDetailSchema, X as runWithEvalClock, _t as validateEvalTagName, bt as runSummarySchema, d as loadEvalModule, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, g as commitPendingCacheWrites, gt as matchesTagsFilter, ht as dedupeEvalTags, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromChildStatuses, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveStatusFromCaseRows, q as runInEvalRuntimeScope, t as filterEvalCases, u as runWithModuleIsolation, vt as validateTagsFilterExpression, wt as getCaseRowCaseKey, yt as runManifestSchema } from "./runExecution-BH7DlMXl.mjs";
+import { Et as caseRowSchema, Pt as runWithEvalRegistry, Tt as caseDetailSchema, X as runWithEvalClock, _t as validateEvalTagName, bt as runSummarySchema, d as loadEvalModule, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, g as commitPendingCacheWrites, gt as matchesTagsFilter, ht as dedupeEvalTags, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromChildStatuses, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveStatusFromCaseRows, q as runInEvalRuntimeScope, t as filterEvalCases, u as runWithModuleIsolation, vt as validateTagsFilterExpression, wt as getCaseRowCaseKey, yt as runManifestSchema } from "./runExecution-C4kAOhC1.mjs";
 import { readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { existsSync } from "node:fs";

package/dist/{runner-DbVYcapC.mjs → runner-JIykMlve.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-B3hEOT_I.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-bjd_UB9i.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-B3hEOT_I.mjs → runner-bjd_UB9i.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-Ck0mqxd-.mjs";
-import "./src-B3iq-tuv.mjs";
+import { n as createRunner } from "./cli-Cf37PZKi.mjs";
+import "./src-303BocMW.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{src-B3iq-tuv.mjs → src-303BocMW.mjs}

@@ -1,5 +1,5 @@
-import { Mt as defineEval$1, W as matchesEvalTags$1 } from "./runExecution-BH7DlMXl.mjs";
-import "./cli-Ck0mqxd-.mjs";
+import { Mt as defineEval$1, W as matchesEvalTags$1 } from "./runExecution-C4kAOhC1.mjs";
+import "./cli-Cf37PZKi.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.57.0",
+  "version": "0.58.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"

package/skills/agent-eval/SKILL.md

@@ -626,9 +626,11 @@ When adding or changing evals:
 1. Put the tracing + ambient SDK calls in the product code that runs in both
    production and evals. Keep eval files thin.
 2. Use realistic cases drawn from real product flows; avoid placeholder inputs.
-3. `evalAssert` for hard invariants and truthy type narrowing, `evalExpect`
-   for non-trivial comparisons, `scores` for graded signals, `passThreshold`
-   only on scores that should gate pass/fail.
+3. `evalAssert` for hard invariants and truthy type narrowing. It records
+   pass/fail entries in case-detail `assertions`; failed entries are also kept
+   in `assertionFailures` and fail the case. Use `evalExpect` for non-trivial
+   comparisons, `scores` for graded signals, and `passThreshold` only on
+   scores that should gate pass/fail.
 4. Surface reviewable values through execute-context `setOutput` or ambient
    `setEvalOutput` in shared workflow code, and shape them with `columns`
    formats from the `ColumnFormat` type.