@ls-stack/agent-eval 0.58.2 → 0.58.4

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as setScopeCacheContext, A as repoFile, B as evalTime, C as evalTracer, D as deserializeCacheValue, E as deserializeCacheRecording, F as EvalRuntimeUsageError, Ft as getEvalRegistry, H as getEvalCaseInput, I as appendToEvalOutput, J as runInEvalRuntimeScope, K as mergeEvalOutput, M as readManualInputFile, N as evalExpect, O as serializeCacheRecording, P as EvalAssertionError, Q as setEvalOutput, R as evalAssert, S as evalSpan, T as hashCacheKeySync, U as incrementEvalOutput, V as getCurrentScope, W as isInEvalScope, X as runInExistingEvalScope, Y as runInEvalScope, at as extractApiCalls, b as buildTraceTree, ct as simulateTokenAllocation, et as startEvalBackgroundJob, it as extractCacheHits, j as manualInputFileValueSchema, k as serializeCacheValue, ot as extractLlmCalls, q as nextEvalId, rt as extractCacheEntries, st as simulateLlmCallCost, ut as getNestedAttribute, w as hashCacheKey, x as captureEvalSpanError, y as z, z as evalLog } from "./runExecution-pHJ0_TzH.mjs";
-import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-HBwXIJsg.mjs";
-import { n as matchesEvalTags, t as defineEval } from "./src-AeXGBJ26.mjs";
+import { $ as setEvalOutput, A as serializeCacheValue, B as evalLog, C as evalSpan, D as deserializeCacheRecording, E as hashCacheKeySync, F as EvalAssertionError, G as isInEvalScope, H as getCurrentScope, I as EvalRuntimeUsageError, It as getEvalRegistry, J as nextEvalId, L as appendToEvalOutput, M as manualInputFileValueSchema, N as readManualInputFile, O as deserializeCacheValue, P as evalExpect, S as captureEvalSpanError, T as hashCacheKey, U as getEvalCaseInput, V as evalTime, W as incrementEvalOutput, X as runInEvalScope, Y as runInEvalRuntimeScope, Z as runInExistingEvalScope, at as extractCacheHits, b as z, ct as simulateLlmCallCost, dt as getNestedAttribute, et as setScopeCacheContext, it as extractCacheEntries, j as repoFile, k as serializeCacheRecording, lt as simulateTokenAllocation, ot as extractApiCalls, q as mergeEvalOutput, st as extractLlmCalls, tt as startEvalBackgroundJob, w as evalTracer, x as buildTraceTree, z as evalAssert } from "./runExecution-CLkC-4Z1.mjs";
+import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-Bf5RzM8O.mjs";
+import { n as matchesEvalTags, t as defineEval } from "./src-BjMMDm_O.mjs";
 export { EvalAssertionError, EvalRuntimeUsageError, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, z };

package/dist/runChild.mjs

@@ -1,5 +1,5 @@
-import { At as evalStatsConfigSchema, L as configureEvalRunLogs, Mt as evalChartsConfigSchema, Nt as columnDefSchema, Tt as buildEvalKey, bt as runManifestSchema, jt as manualInputDescriptorSchema, kt as evalStatAggregateSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, tt as createRunRequestSchema, v as createFsCacheStore, xt as runSummarySchema } from "./runExecution-pHJ0_TzH.mjs";
-import { S as parseEvalDiscovery, m as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-ngVXShH4.mjs";
+import { At as evalStatAggregateSchema, Et as buildEvalKey, Mt as manualInputDescriptorSchema, Nt as evalChartsConfigSchema, Pt as columnDefSchema, R as configureEvalRunLogs, St as runSummarySchema, jt as evalStatsConfigSchema, l as registerAgentEvalsPackageResolutionHooks, nt as createRunRequestSchema, p as loadConfig, v as createFsCacheStore, xt as runManifestSchema, y as getCacheRetentionOptions } from "./runExecution-CLkC-4Z1.mjs";
+import { S as parseEvalDiscovery, m as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-BS-WxTee.mjs";
 import { z } from "zod/v4";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";
@@ -137,11 +137,12 @@ async function main() {
 	registerAgentEvalsPackageResolutionHooks();
 	const config = await loadConfig();
 	configureEvalRunLogs({ captureConsole: config.runLogs?.captureConsole !== false });
+	const cacheRetentionOptions = getCacheRetentionOptions(config.cache);
 	const cacheStore = createFsCacheStore({
 		workspaceRoot: context.workspaceRoot,
 		dir: config.cache?.dir,
-		maxEntriesPerNamespace: config.cache?.maxEntriesPerNamespace ?? config.cache?.maxEntriesPerEval,
-		maxEntriesByNamespace: config.cache?.maxEntriesByNamespace,
+		maxEntriesPerNamespace: cacheRetentionOptions.maxEntriesPerNamespace,
+		maxEntriesByNamespace: cacheRetentionOptions.maxEntriesByNamespace,
 		lastAccessedAtUpdateIntervalMs: config.cache?.lastAccessedAtUpdateIntervalMs
 	});
 	const evalMetas = await discoverRunEvals({

package/dist/{runExecution-pHJ0_TzH.mjs → runExecution-CLkC-4Z1.mjs}

@@ -1009,9 +1009,8 @@ const removeDefaultConfigSchema = z.union([z.literal(true), z.array(defaultConfi
 const evalDeriveValueFnSchema = z.custom((value) => typeof value === "function", { message: "Expected a derive output function" });
 /** Schema for keyed or object-returning trace-derived output config. */
 const evalDeriveConfigSchema = z.union([z.custom((value) => typeof value === "function", { message: "Expected a deriveFromTracing function" }), z.record(z.string().min(1), evalDeriveValueFnSchema)]);
-const evalTracingAssertionsFnSchema = z.custom((value) => typeof value === "function", { message: "Expected a tracing assertions function" });
-/** Schema for function or keyed trace-derived assertion config. */
-const evalTracingAssertionsConfigSchema = z.union([z.custom((value) => typeof value === "function", { message: "Expected a tracingAssertions function" }), z.record(z.string().min(1), evalTracingAssertionsFnSchema)]);
+/** Schema for trace-derived assertion config. */
+const evalTracingAssertionsConfigSchema = z.custom((value) => typeof value === "function", { message: "Expected a tracingAssertions function" });
 /** Schema for UI overrides on derived or scored columns. */
 const evalColumnOverrideSchema = z.object({
 	label: z.string().optional(),
@@ -1402,6 +1401,10 @@ function resolveApiCallsConfig(input) {
 		metrics: (input?.metrics ?? []).map(resolveApiCallMetric)
 	};
 }
+const cacheMaxEntriesSchema = z.union([z.number(), z.object({
+	default: z.number().optional(),
+	namespaces: z.record(z.string(), z.number()).optional()
+})]).optional();
 /** Zod schema for validating `agent-evals.config.ts` input. */
 const agentEvalsConfigSchema = z.object({
 	workspaceRoot: z.string().optional(),
@@ -1425,11 +1428,26 @@ const agentEvalsConfigSchema = z.object({
 	cache: z.object({
 		enabled: z.boolean().optional(),
 		dir: z.string().optional(),
-		maxEntriesPerNamespace: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
+		maxEntries: cacheMaxEntriesSchema,
+		maxEntriesPerNamespace: z.number().optional(),
 		maxEntriesByNamespace: z.record(z.string(), z.number()).optional(),
 		pruneIdleDelayMs: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
 		lastAccessedAtUpdateIntervalMs: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional()),
-		maxEntriesPerEval: z.preprocess((value) => typeof value === "number" && Number.isFinite(value) ? value : void 0, z.number().optional())
+		maxEntriesPerEval: z.number().optional()
+	}).transform(({ maxEntries, maxEntriesByNamespace, maxEntriesPerEval, maxEntriesPerNamespace, ...cache }) => {
+		const defaultMaxEntries = maxEntriesPerNamespace ?? maxEntriesPerEval;
+		if (maxEntries !== void 0) return {
+			...cache,
+			maxEntries
+		};
+		if (defaultMaxEntries !== void 0 || maxEntriesByNamespace !== void 0) return {
+			...cache,
+			maxEntries: {
+				default: defaultMaxEntries,
+				namespaces: maxEntriesByNamespace
+			}
+		};
+		return cache;
 	}).optional()
 });
 //#endregion
@@ -5079,6 +5097,19 @@ function buildTraceTree(spans, checkpoints) {
 	};
 }
 //#endregion
+//#region ../runner/src/cacheConfig.ts
+function getCacheRetentionOptions(cacheConfig) {
+	const maxEntries = cacheConfig?.maxEntries;
+	if (typeof maxEntries === "number") return {
+		maxEntriesPerNamespace: maxEntries,
+		maxEntriesByNamespace: void 0
+	};
+	return {
+		maxEntriesPerNamespace: maxEntries?.default,
+		maxEntriesByNamespace: maxEntries?.namespaces
+	};
+}
+//#endregion
 //#region ../runner/src/cacheAccessTime.ts
 const defaultLastAccessedAtUpdateIntervalMs = 14400 * 1e3;
 function normalizeLastAccessedAtUpdateIntervalMs(value) {
@@ -6754,19 +6785,9 @@ async function runOneTracingAssertion(params) {
 }
 async function runTracingAssertionsConfig(params) {
 	if (params.tracingAssertions === void 0) return;
-	if (typeof params.tracingAssertions === "function") {
-		await runOneTracingAssertion({
-			label: "tracingAssertions",
-			tracingAssertion: params.tracingAssertions,
-			scope: params.scope,
-			traceTree: params.traceTree,
-			evalCase: params.evalCase
-		});
-		return;
-	}
-	for (const [key, tracingAssertion] of Object.entries(params.tracingAssertions)) await runOneTracingAssertion({
-		label: `tracingAssertions "${key}"`,
-		tracingAssertion,
+	await runOneTracingAssertion({
+		label: "tracingAssertions",
+		tracingAssertion: params.tracingAssertions,
 		scope: params.scope,
 		traceTree: params.traceTree,
 		evalCase: params.evalCase
@@ -7037,4 +7058,4 @@ function recordAssertionFailure(scope, failure) {
 	});
 }
 //#endregion
-export { setScopeCacheContext as $, repoFile as A, evalStatsConfigSchema as At, evalTime as B, evalTracer as C, resolveLlmCallsConfig as Ct, deserializeCacheValue as D, caseDetailSchema as Dt, deserializeCacheRecording as E, getCaseRowCaseKey as Et, EvalRuntimeUsageError as F, getEvalRegistry as Ft, matchesEvalTags as G, getEvalCaseInput as H, appendToEvalOutput as I, runWithEvalRegistry as It, runInEvalRuntimeScope as J, mergeEvalOutput as K, configureEvalRunLogs as L, readManualInputFile as M, evalChartsConfigSchema as Mt, evalExpect as N, columnDefSchema as Nt, serializeCacheRecording as O, caseRowSchema as Ot, EvalAssertionError as P, defineEval as Pt, setEvalOutput as Q, evalAssert as R, evalSpan as S, resolveApiCallsConfig as St, hashCacheKeySync as T, buildEvalKey as Tt, incrementEvalOutput as U, getCurrentScope as V, isInEvalScope as W, runInExistingEvalScope as X, runInEvalScope as Y, runWithEvalClock as Z, createBufferedCacheStore as _, matchesTagsFilter as _t, isCaseChildParentMessage as a, extractApiCalls as at, buildTraceTree as b, runManifestSchema as bt, resolveArtifactPath as c, simulateTokenAllocation as ct, loadEvalModule as d, getEvalTitle as dt, startEvalBackgroundJob as et, resolveEvalDefaultConfig as f, getEvalDisplayStatus as ft, commitPendingCacheWrites as g, dedupeEvalTags as gt, normalizeScoreDef as h, deriveStatusFromChildStatuses as ht, isCaseChildMessage as i, extractCacheHits as it, manualInputFileValueSchema as j, manualInputDescriptorSchema as jt, serializeCacheValue as k, evalStatAggregateSchema as kt, registerAgentEvalsPackageResolutionHooks as l, applyDerivedCallAttributes as lt, buildDeclaredColumnDefs as m, deriveStatusFromCaseRows as mt, resolveRunnableEvalCases as n, updateManualScoreRequestSchema as nt, stripTerminalControlCodes as o, extractLlmCalls as ot, loadConfig as p, deriveScopedSummaryFromCases as pt, nextEvalId as q, runCase as r, extractCacheEntries as rt, resolveTracePresentation as s, simulateLlmCallCost as st, filterEvalCases as t, createRunRequestSchema as tt, runWithModuleIsolation as u, getNestedAttribute as ut, createFsCacheStore as v, validateEvalTagName as vt, hashCacheKey as w, buildCaseKey as wt, captureEvalSpanError as x, runSummarySchema as xt, z$1 as y, validateTagsFilterExpression as yt, evalLog as z };
+export { setEvalOutput as $, serializeCacheValue as A, evalStatAggregateSchema as At, evalLog as B, evalSpan as C, resolveApiCallsConfig as Ct, deserializeCacheRecording as D, getCaseRowCaseKey as Dt, hashCacheKeySync as E, buildEvalKey as Et, EvalAssertionError as F, defineEval as Ft, isInEvalScope as G, getCurrentScope as H, EvalRuntimeUsageError as I, getEvalRegistry as It, nextEvalId as J, matchesEvalTags as K, appendToEvalOutput as L, runWithEvalRegistry as Lt, manualInputFileValueSchema as M, manualInputDescriptorSchema as Mt, readManualInputFile as N, evalChartsConfigSchema as Nt, deserializeCacheValue as O, caseDetailSchema as Ot, evalExpect as P, columnDefSchema as Pt, runWithEvalClock as Q, configureEvalRunLogs as R, captureEvalSpanError as S, runSummarySchema as St, hashCacheKey as T, buildCaseKey as Tt, getEvalCaseInput as U, evalTime as V, incrementEvalOutput as W, runInEvalScope as X, runInEvalRuntimeScope as Y, runInExistingEvalScope as Z, createBufferedCacheStore as _, dedupeEvalTags as _t, isCaseChildParentMessage as a, extractCacheHits as at, z$1 as b, validateTagsFilterExpression as bt, resolveArtifactPath as c, simulateLlmCallCost as ct, loadEvalModule as d, getNestedAttribute as dt, setScopeCacheContext as et, resolveEvalDefaultConfig as f, getEvalTitle as ft, commitPendingCacheWrites as g, deriveStatusFromChildStatuses as gt, normalizeScoreDef as h, deriveStatusFromCaseRows as ht, isCaseChildMessage as i, extractCacheEntries as it, repoFile as j, evalStatsConfigSchema as jt, serializeCacheRecording as k, caseRowSchema as kt, registerAgentEvalsPackageResolutionHooks as l, simulateTokenAllocation as lt, buildDeclaredColumnDefs as m, deriveScopedSummaryFromCases as mt, resolveRunnableEvalCases as n, createRunRequestSchema as nt, stripTerminalControlCodes as o, extractApiCalls as ot, loadConfig as p, getEvalDisplayStatus as pt, mergeEvalOutput as q, runCase as r, updateManualScoreRequestSchema as rt, resolveTracePresentation as s, extractLlmCalls as st, filterEvalCases as t, startEvalBackgroundJob as tt, runWithModuleIsolation as u, applyDerivedCallAttributes as ut, createFsCacheStore as v, matchesTagsFilter as vt, evalTracer as w, resolveLlmCallsConfig as wt, buildTraceTree as x, runManifestSchema as xt, getCacheRetentionOptions as y, validateEvalTagName as yt, evalAssert as z };

package/dist/{runOrchestration-ngVXShH4.mjs → runOrchestration-BS-WxTee.mjs}

@@ -1,4 +1,4 @@
-import { Dt as caseDetailSchema, Et as getCaseRowCaseKey, It as runWithEvalRegistry, J as runInEvalRuntimeScope, Ot as caseRowSchema, Z as runWithEvalClock, _t as matchesTagsFilter, bt as runManifestSchema, d as loadEvalModule, f as resolveEvalDefaultConfig, g as commitPendingCacheWrites, gt as dedupeEvalTags, ht as deriveStatusFromChildStatuses, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromCaseRows, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveScopedSummaryFromCases, t as filterEvalCases, u as runWithModuleIsolation, vt as validateEvalTagName, wt as buildCaseKey, xt as runSummarySchema, yt as validateTagsFilterExpression } from "./runExecution-pHJ0_TzH.mjs";
+import { Dt as getCaseRowCaseKey, Lt as runWithEvalRegistry, Ot as caseDetailSchema, Q as runWithEvalClock, St as runSummarySchema, Tt as buildCaseKey, Y as runInEvalRuntimeScope, _t as dedupeEvalTags, bt as validateTagsFilterExpression, d as loadEvalModule, f as resolveEvalDefaultConfig, g as commitPendingCacheWrites, gt as deriveStatusFromChildStatuses, ht as deriveStatusFromCaseRows, i as isCaseChildMessage, kt as caseRowSchema, m as buildDeclaredColumnDefs, mt as deriveScopedSummaryFromCases, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, t as filterEvalCases, u as runWithModuleIsolation, vt as matchesTagsFilter, xt as runManifestSchema, yt as validateEvalTagName } from "./runExecution-CLkC-4Z1.mjs";
 import { readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { existsSync } from "node:fs";

package/dist/{runner-BnZMGBla.mjs → runner-Bz5ZPqmm.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-D_pz2NON.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-DW-11txl.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-D_pz2NON.mjs → runner-DW-11txl.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-HBwXIJsg.mjs";
-import "./src-AeXGBJ26.mjs";
+import { n as createRunner } from "./cli-Bf5RzM8O.mjs";
+import "./src-BjMMDm_O.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{src-AeXGBJ26.mjs → src-BjMMDm_O.mjs}

@@ -1,5 +1,5 @@
-import { G as matchesEvalTags$1, Pt as defineEval$1 } from "./runExecution-pHJ0_TzH.mjs";
-import "./cli-HBwXIJsg.mjs";
+import { Ft as defineEval$1, K as matchesEvalTags$1 } from "./runExecution-CLkC-4Z1.mjs";
+import "./cli-Bf5RzM8O.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.58.2",
+  "version": "0.58.4",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"

package/skills/agent-eval/SKILL.md

@@ -366,10 +366,11 @@ See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape
   form, return `undefined` to omit one output for that case. Do not call
   `evalAssert(...)` or `evalExpect(...)` from `deriveFromTracing`; use
   `tracingAssertions` for trace-derived pass/fail checks.
-- `tracingAssertions` can be authored globally or locally on one eval when a
-  finished-trace invariant should pass or fail the case without creating a fake
-  score column. It receives the same `{ trace, input, case }` context as
-  `deriveFromTracing`; call `evalAssert(...)` or `evalExpect(...)` inside it.
+- `tracingAssertions` is a single function that can be authored globally or
+  locally on one eval when a finished-trace invariant should pass or fail the
+  case without creating a fake score column. It receives the same
+  `{ trace, input, case }` context as `deriveFromTracing`; call
+  `evalAssert(...)` or `evalExpect(...)` inside it.
   Useful trace helpers include `trace.findSpan(name)`, `trace.findSpans(name)`,
   `trace.hasSpan(name)`, `trace.findSpansByKind(kind)`,
   `trace.findToolCallSpans()`, `trace.listToolCallSpanNames()`,
@@ -545,8 +546,8 @@ Mental model:
   JSON blob refs. Each namespace is capped at 100 entries by default. The runner
   prunes least recently accessed indexed entries after a run finishes and the
   runner stays idle for `cache.pruneIdleDelayMs ?? 5000` milliseconds. Configure
-  `cache.maxEntriesPerNamespace` for the default cap and
-  `cache.maxEntriesByNamespace` for exact namespace-specific caps.
+  `cache.maxEntries` as a number for the default cap, or as
+  `{ default, namespaces }` for exact namespace-specific caps.
 - Unindexed legacy cache files are ignored by normal lookup/listing. Use
   `agent-evals cache repair` to remove unindexed cache files, stale index rows,
   debug sidecars, and unreferenced blob files.