@ls-stack/agent-eval 0.60.2 → 0.60.4

package/dist/{app-DPCFFkyQ.mjs → app-gg10KvzS.mjs}

@@ -1,7 +1,7 @@
-import { Et as getCaseRowCaseKey, Ot as caseRowSchema, dt as getEvalTitle, nt as updateManualScoreRequestSchema, rt as extractCacheEntries, tt as createRunRequestSchema } from "./runExecution-Bq0Y3y_1.mjs";
-import { o as stageManualInputFile } from "./cli-CbePEEua.mjs";
-import "./src-CVM_FqPx.mjs";
-import { t as getRunnerInstance } from "./runner-Kp0JqxrU.mjs";
+import { Dt as caseRowSchema, Tt as getCaseRowCaseKey, et as createRunRequestSchema, nt as extractCacheEntries, tt as updateManualScoreRequestSchema, ut as getEvalTitle } from "./runExecution-Bu9yfdUS.mjs";
+import { o as stageManualInputFile } from "./cli-OLZIjQpx.mjs";
+import "./src-Cy3OxoZW.mjs";
+import { t as getRunnerInstance } from "./runner-SxtKn-Xh.mjs";
 import { z } from "zod";
 import { resultify } from "t-result";
 import { readFile } from "node:fs/promises";

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-CbePEEua.mjs";
+import { t as runCli } from "./cli-OLZIjQpx.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/caseChild.mjs

@@ -1,4 +1,4 @@
-import { Ct as resolveLlmCallsConfig, It as runWithEvalRegistry, J as runInEvalRuntimeScope, L as configureEvalRunLogs, St as resolveApiCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, r as runCase, v as createFsCacheStore, y as getCacheRetentionOptions } from "./runExecution-Bq0Y3y_1.mjs";
+import { Ft as runWithEvalRegistry, I as configureEvalRunLogs, St as resolveLlmCallsConfig, _ as createFsCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, g as createBufferedCacheStore, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as getCacheRetentionOptions, xt as resolveApiCallsConfig } from "./runExecution-Bu9yfdUS.mjs";
 //#region ../runner/src/caseChild.ts
 let fatalErrorReported = false;
 let disconnectExpected = false;

package/dist/{cli-CbePEEua.mjs → cli-OLZIjQpx.mjs}

@@ -1,5 +1,5 @@
-import { Ct as resolveLlmCallsConfig, Et as getCaseRowCaseKey, Ft as getEvalRegistry, Ot as caseRowSchema, St as resolveApiCallsConfig, Tt as buildEvalKey, _t as matchesTagsFilter, c as resolveArtifactPath, dt as getEvalTitle, f as resolveEvalDefaultConfig, ft as getEvalDisplayStatus, h as normalizeScoreDef, lt as applyDerivedCallAttributes, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, pt as deriveScopedSummaryFromCases, s as resolveTracePresentation, v as createFsCacheStore, xt as runSummarySchema, y as getCacheRetentionOptions } from "./runExecution-Bq0Y3y_1.mjs";
-import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-BpwW0AmB.mjs";
+import { Dt as caseRowSchema, Pt as getEvalRegistry, St as resolveLlmCallsConfig, Tt as getCaseRowCaseKey, _ as createFsCacheStore, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as getCacheRetentionOptions, wt as buildEvalKey, xt as resolveApiCallsConfig } from "./runExecution-Bu9yfdUS.mjs";
+import { C as parseEvalDiscovery, S as loadIsolatedEvalRegistry, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as parseManualInputValues, c as getLatestRunInfos, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, l as loadPersistedCaseDetail, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as generateRunId, p as persistCaseDetail, s as getLastRunStatuses, u as loadPersistedRunSnapshot, v as runTouchesEval, w as validateCharts, x as deriveEvalFreshness, y as buildManualInputDescriptor } from "./runOrchestration-mpgZmEZ6.mjs";
 import { parseEnv } from "node:util";
 import { resultify } from "t-result";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
@@ -28,10 +28,14 @@ function resolveCaseDetailLookup(run, caseId) {
 		const caseDetail = run.caseDetails.get(lookupId);
 		if (caseDetail) return caseDetail;
 	}
-	const matchingCaseRow = run.cases.find((caseRow) => lookupIds.has(getCaseRowCaseKey(caseRow)) || lookupIds.has(caseRow.caseId));
+	const matchingCaseRow = resolveCaseRowForCaseDetailLookup(run, caseId);
 	if (matchingCaseRow === void 0) return void 0;
 	return run.caseDetails.get(getCaseRowCaseKey(matchingCaseRow));
 }
+function resolveCaseRowForCaseDetailLookup(run, caseId) {
+	const lookupIds = new Set(getCaseLookupIds(caseId));
+	return run.cases.find((caseRow) => lookupIds.has(getCaseRowCaseKey(caseRow)) || lookupIds.has(caseRow.caseId));
+}
 //#endregion
 //#region ../runner/src/configReload.ts
 /** Coordinates idle-only reloads for workspace config and `.env` in app mode. */
@@ -823,7 +827,7 @@ async function markRunTerminalFromChild(runState, event, managerContext) {
 		runState.manifest = snapshot.manifest;
 		runState.summary = snapshot.summary;
 		runState.cases = snapshot.cases;
-		runState.caseDetails = snapshot.caseDetails;
+		runState.caseDetails = /* @__PURE__ */ new Map();
 	} else if (event.type === "run.finished") {
 		runState.manifest.status = "completed";
 		runState.manifest.endedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -969,7 +973,6 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 	let discoveryRefreshTimer;
 	let runHistoryRefreshTimer;
 	let cachePruneIdleTimer;
-	let registryLoadCounter = 0;
 	const configReload = createConfigReloadController({
 		getActiveRunCount,
 		closeRunnerWatchers: closeWatchers,
@@ -995,9 +998,28 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 		if (typeof config.concurrency !== "number" || !Number.isFinite(config.concurrency)) return 1;
 		return Math.max(1, Math.floor(config.concurrency));
 	}
-	function nextRegistryLoadIsolationKey(prefix, filePath) {
-		registryLoadCounter++;
-		return `${prefix}:${String(registryLoadCounter)}:${filePath}`;
+	function getCaseDetailFileId(run, caseRow) {
+		const caseKey = getCaseRowCaseKey(caseRow);
+		return run.cases.some((existing) => existing.caseId === caseRow.caseId && getCaseRowCaseKey(existing) !== caseKey) ? caseKey : caseRow.caseId;
+	}
+	function hydrateCaseDetailForRow(run, caseRow) {
+		const caseKey = getCaseRowCaseKey(caseRow);
+		const cached = run.caseDetails.get(caseKey);
+		if (cached !== void 0) return cached;
+		const detail = loadPersistedCaseDetail(run.runDir, getCaseDetailFileId(run, caseRow));
+		if (detail === null) return void 0;
+		run.caseDetails.set(detail.caseKey ?? detail.caseId, detail);
+		return detail;
+	}
+	function hydrateCaseDetailForLookup(run, caseId) {
+		const cached = resolveCaseDetailLookup(run, caseId);
+		if (cached !== void 0) return cached;
+		const caseRow = resolveCaseRowForCaseDetailLookup(run, caseId);
+		if (caseRow === void 0) return void 0;
+		return hydrateCaseDetailForRow(run, caseRow);
+	}
+	function getDiscoveryModuleIsolationKey(filePath) {
+		return `discovery:${filePath}`;
 	}
 	const runner = {
 		async init() {
@@ -1018,29 +1040,17 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 		async recomputeStatusesForEval(evalKey) {
 			const evalMeta = resolveEvalMeta(evalKey);
 			if (!evalMeta) return { updatedRuns: 0 };
-			const entry = (await loadIsolatedEvalRegistry({
-				evalFilePath: evalMeta.sourceFilePath,
-				sourceFingerprint: evalMeta.sourceFingerprint ?? void 0,
-				moduleIsolation: {
-					key: nextRegistryLoadIsolationKey("recompute-status", evalMeta.sourceFilePath),
-					workspaceRoot
-				},
-				runtimeScope: "env"
-			})).get(evalMeta.id);
-			if (!entry) return { updatedRuns: 0 };
 			const scoreThresholds = /* @__PURE__ */ new Map();
-			entry.use((evalDef) => {
-				for (const [key, def] of Object.entries(evalDef.scores ?? {})) {
-					const threshold = normalizeScoreDef(def).passThreshold;
-					if (threshold !== void 0) scoreThresholds.set(key, threshold);
-				}
-				for (const [key, def] of Object.entries(evalDef.manualScores ?? {})) if (def.passThreshold !== void 0) scoreThresholds.set(key, def.passThreshold);
-			});
+			for (const columnDef of evalMeta.columnDefs) {
+				if (columnDef.isScore !== true || columnDef.passThreshold === void 0) continue;
+				scoreThresholds.set(columnDef.key, columnDef.passThreshold);
+			}
 			const updatedRuns = await recomputeEvalStatusesInRuns({
 				runs: runs.values(),
 				evalKey: evalMeta.key,
 				evalExists: evals.has(evalMeta.key),
 				scoreThresholds,
+				getCaseDetail: hydrateCaseDetailForRow,
 				persistCaseDetail
 			});
 			emitDiscoveryEvent();
@@ -1052,6 +1062,7 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 				updated: false,
 				reason: "Run not found"
 			};
+			hydrateCaseDetailForLookup(run, caseId);
 			return recalculateDerivedAttributesForCase({
 				run,
 				caseId,
@@ -1107,7 +1118,7 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 				updated: false,
 				reason: "Manual score not found"
 			};
-			const caseDetail = run.caseDetails.get(getCaseRowCaseKey(caseRow));
+			const caseDetail = hydrateCaseDetailForRow(run, caseRow);
 			if (!caseDetail) return {
 				updated: false,
 				reason: "Case detail not found"
@@ -1238,7 +1249,7 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 						evalFilePath: filePath,
 						sourceFingerprint,
 						moduleIsolation: {
-							key: nextRegistryLoadIsolationKey("discovery", filePath),
+							key: getDiscoveryModuleIsolationKey(filePath),
 							workspaceRoot
 						},
 						runtimeScope: "env"
@@ -1480,7 +1491,7 @@ function createRunner({ watchForChanges = true, loadEnv = true } = {}) {
 		getCaseDetail(runId, caseId) {
 			const run = runs.get(runId);
 			if (!run) return void 0;
-			return resolveCaseDetailLookup(run, caseId);
+			return hydrateCaseDetailForLookup(run, caseId);
 		},
 		subscribe(runId, listener) {
 			const run = runs.get(runId);
@@ -2232,8 +2243,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-DPCFFkyQ.mjs");
-	const runnerModule = await import("./runner-XEP21_u9.mjs");
+	const appModule = await import("./app-gg10KvzS.mjs");
+	const runnerModule = await import("./runner-C4Y0lWb1.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner({ loadEnv: args.loadEnv });

package/dist/index.d.mts

@@ -1942,7 +1942,6 @@ declare const columnFormatSchema: z.ZodEnum<{
   number: "number";
   boolean: "boolean";
   file: "file";
-  duration: "duration";
   markdown: "markdown";
   json: "json";
   image: "image";
@@ -1950,6 +1949,7 @@ declare const columnFormatSchema: z.ZodEnum<{
   pdf: "pdf";
   audio: "audio";
   video: "video";
+  duration: "duration";
   percent: "percent";
   passFail: "passFail";
   stars: "stars";
@@ -1969,7 +1969,6 @@ declare const columnDefSchema: z.ZodObject<{
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
@@ -1977,6 +1976,7 @@ declare const columnDefSchema: z.ZodObject<{
     pdf: "pdf";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -2022,8 +2022,8 @@ type CellValue = z.infer<typeof cellValueSchema>; //#endregion
 declare const traceAttributeDisplayFormatSchema: z.ZodEnum<{
   string: "string";
   number: "number";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /**
  * Formatting hint for trace attribute values rendered by the UI.
@@ -2047,8 +2047,8 @@ declare const traceAttributeDisplaySchema: z.ZodObject<{
   format: z.ZodOptional<z.ZodEnum<{
     string: "string";
     number: "number";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -2083,8 +2083,8 @@ declare const traceDisplayConfigSchema: z.ZodObject<{
     format: z.ZodOptional<z.ZodEnum<{
       string: "string";
       number: "number";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -2123,8 +2123,8 @@ declare const traceAttributeDisplayInputSchema: z.ZodObject<{
   format: z.ZodOptional<z.ZodEnum<{
     string: "string";
     number: "number";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -2161,8 +2161,8 @@ declare const traceDisplayInputConfigSchema: z.ZodObject<{
     format: z.ZodOptional<z.ZodEnum<{
       string: "string";
       number: "number";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -2214,8 +2214,8 @@ declare const traceSpanSchema$1: z.ZodObject<{
   status: z.ZodEnum<{
     error: "error";
     running: "running";
-    cancelled: "cancelled";
     ok: "ok";
+    cancelled: "cancelled";
   }>;
   attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
   error: z.ZodOptional<z.ZodObject<{
@@ -2327,7 +2327,6 @@ declare const evalStatItemSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
@@ -2335,6 +2334,7 @@ declare const evalStatItemSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     pdf: "pdf";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -2391,7 +2391,6 @@ declare const evalStatsConfigSchema: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodOb
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
@@ -2399,6 +2398,7 @@ declare const evalStatsConfigSchema: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodOb
     pdf: "pdf";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -2437,7 +2437,6 @@ declare const evalSummarySchema$1: z.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
@@ -2445,6 +2444,7 @@ declare const evalSummarySchema$1: z.ZodObject<{
       pdf: "pdf";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -2466,10 +2466,10 @@ declare const evalSummarySchema$1: z.ZodObject<{
   caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
   lastRunStatus: z.ZodNullable<z.ZodEnum<{
     error: "error";
-    pass: "pass";
-    fail: "fail";
     running: "running";
     cancelled: "cancelled";
+    pass: "pass";
+    fail: "fail";
     unscored: "unscored";
   }>>;
   stats: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -2518,7 +2518,6 @@ declare const evalSummarySchema$1: z.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
@@ -2526,6 +2525,7 @@ declare const evalSummarySchema$1: z.ZodObject<{
       pdf: "pdf";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -2558,9 +2558,9 @@ declare const evalSummarySchema$1: z.ZodObject<{
       }>;
       label: z.ZodOptional<z.ZodString>;
       color: z.ZodOptional<z.ZodEnum<{
+        error: "error";
         success: "success";
         accent: "accent";
-        error: "error";
         accentDim: "accentDim";
         warning: "warning";
         textMuted: "textMuted";
@@ -2582,9 +2582,9 @@ declare const evalSummarySchema$1: z.ZodObject<{
       }>;
       label: z.ZodOptional<z.ZodString>;
       color: z.ZodOptional<z.ZodEnum<{
+        error: "error";
         success: "success";
         accent: "accent";
-        error: "error";
         accentDim: "accentDim";
         warning: "warning";
         textMuted: "textMuted";
@@ -2715,10 +2715,10 @@ declare const caseRowSchema$1: z.ZodObject<{
   tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
   status: z.ZodEnum<{
     error: "error";
-    pass: "pass";
-    fail: "fail";
     running: "running";
     cancelled: "cancelled";
+    pass: "pass";
+    fail: "fail";
     pending: "pending";
   }>;
   durationMs: z.ZodNullable<z.ZodNumber>;
@@ -2749,7 +2749,6 @@ declare const caseRowSchema$1: z.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
@@ -2757,6 +2756,7 @@ declare const caseRowSchema$1: z.ZodObject<{
       pdf: "pdf";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -2857,8 +2857,8 @@ declare const scoreTraceSchema: z.ZodObject<{
     status: z.ZodEnum<{
       error: "error";
       running: "running";
-      cancelled: "cancelled";
       ok: "ok";
+      cancelled: "cancelled";
     }>;
     attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     error: z.ZodOptional<z.ZodObject<{
@@ -2894,8 +2894,8 @@ declare const scoreTraceSchema: z.ZodObject<{
       format: z.ZodOptional<z.ZodEnum<{
         string: "string";
         number: "number";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -2920,10 +2920,10 @@ declare const scoreTraceSchema: z.ZodObject<{
     namespace: z.ZodString;
     key: z.ZodString;
     status: z.ZodEnum<{
+      bypass: "bypass";
+      refresh: "refresh";
       hit: "hit";
       miss: "miss";
-      refresh: "refresh";
-      bypass: "bypass";
     }>;
     read: z.ZodOptional<z.ZodBoolean>;
     stored: z.ZodOptional<z.ZodBoolean>;
@@ -2942,10 +2942,10 @@ declare const caseDetailSchema$1: z.ZodObject<{
   tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
   status: z.ZodEnum<{
     error: "error";
-    pass: "pass";
-    fail: "fail";
     running: "running";
     cancelled: "cancelled";
+    pass: "pass";
+    fail: "fail";
     pending: "pending";
   }>;
   input: z.ZodUnknown;
@@ -2960,8 +2960,8 @@ declare const caseDetailSchema$1: z.ZodObject<{
     status: z.ZodEnum<{
       error: "error";
       running: "running";
-      cancelled: "cancelled";
       ok: "ok";
+      cancelled: "cancelled";
     }>;
     attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     error: z.ZodOptional<z.ZodObject<{
@@ -2997,8 +2997,8 @@ declare const caseDetailSchema$1: z.ZodObject<{
       format: z.ZodOptional<z.ZodEnum<{
         string: "string";
         number: "number";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -3029,8 +3029,8 @@ declare const caseDetailSchema$1: z.ZodObject<{
       status: z.ZodEnum<{
         error: "error";
         running: "running";
-        cancelled: "cancelled";
         ok: "ok";
+        cancelled: "cancelled";
       }>;
       attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
       error: z.ZodOptional<z.ZodObject<{
@@ -3066,8 +3066,8 @@ declare const caseDetailSchema$1: z.ZodObject<{
         format: z.ZodOptional<z.ZodEnum<{
           string: "string";
           number: "number";
-          duration: "duration";
           json: "json";
+          duration: "duration";
         }>>;
         numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
         placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -3092,10 +3092,10 @@ declare const caseDetailSchema$1: z.ZodObject<{
       namespace: z.ZodString;
       key: z.ZodString;
       status: z.ZodEnum<{
+        bypass: "bypass";
+        refresh: "refresh";
         hit: "hit";
         miss: "miss";
-        refresh: "refresh";
-        bypass: "bypass";
       }>;
       read: z.ZodOptional<z.ZodBoolean>;
       stored: z.ZodOptional<z.ZodBoolean>;
@@ -3127,7 +3127,6 @@ declare const caseDetailSchema$1: z.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
@@ -3135,6 +3134,7 @@ declare const caseDetailSchema$1: z.ZodObject<{
       pdf: "pdf";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -3213,10 +3213,10 @@ declare const caseDetailSchema$1: z.ZodObject<{
     namespace: z.ZodString;
     key: z.ZodString;
     status: z.ZodEnum<{
+      bypass: "bypass";
+      refresh: "refresh";
       hit: "hit";
       miss: "miss";
-      refresh: "refresh";
-      bypass: "bypass";
     }>;
     read: z.ZodOptional<z.ZodBoolean>;
     stored: z.ZodOptional<z.ZodBoolean>;
@@ -3283,9 +3283,9 @@ type EvalChartAggregate = z.infer<typeof evalChartAggregateSchema>;
  * not emit raw hex so authored evals stay decoupled from the web theme.
  */
 declare const evalChartColorSchema: z.ZodEnum<{
+  error: "error";
   success: "success";
   accent: "accent";
-  error: "error";
   accentDim: "accentDim";
   warning: "warning";
   textMuted: "textMuted";
@@ -3312,9 +3312,9 @@ declare const evalChartMetricSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }>;
   label: z.ZodOptional<z.ZodString>;
   color: z.ZodOptional<z.ZodEnum<{
+    error: "error";
     success: "success";
     accent: "accent";
-    error: "error";
     accentDim: "accentDim";
     warning: "warning";
     textMuted: "textMuted";
@@ -3336,9 +3336,9 @@ declare const evalChartMetricSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }>;
   label: z.ZodOptional<z.ZodString>;
   color: z.ZodOptional<z.ZodEnum<{
+    error: "error";
     success: "success";
     accent: "accent";
-    error: "error";
     accentDim: "accentDim";
     warning: "warning";
     textMuted: "textMuted";
@@ -3395,9 +3395,9 @@ declare const evalChartConfigSchema: z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       success: "success";
       accent: "accent";
-      error: "error";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -3419,9 +3419,9 @@ declare const evalChartConfigSchema: z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       success: "success";
       accent: "accent";
-      error: "error";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -3485,9 +3485,9 @@ declare const evalChartsConfigSchema: z.ZodArray<z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       success: "success";
       accent: "accent";
-      error: "error";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -3509,9 +3509,9 @@ declare const evalChartsConfigSchema: z.ZodArray<z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       success: "success";
       accent: "accent";
-      error: "error";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -3589,9 +3589,9 @@ declare const runManifestSchema$1: z.ZodObject<{
     median: "median";
   }>>>;
   cacheMode: z.ZodOptional<z.ZodEnum<{
-    refresh: "refresh";
-    bypass: "bypass";
     use: "use";
+    bypass: "bypass";
+    refresh: "refresh";
   }>>;
 }, z.core.$strip>;
 /** Persisted lifecycle metadata for a single eval run. */
@@ -3808,8 +3808,8 @@ declare const llmCallMetricFormatSchema$1: z.ZodEnum<{
   string: "string";
   number: "number";
   boolean: "boolean";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /** Render format applied to an LLM-call metric value. */
 type LlmCallMetricFormat = z.infer<typeof llmCallMetricFormatSchema$1>;
@@ -3818,8 +3818,8 @@ declare const apiCallMetricFormatSchema$1: z.ZodEnum<{
   string: "string";
   number: "number";
   boolean: "boolean";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /** Render format applied to an API-call metric value. */
 type ApiCallMetricFormat = z.infer<typeof apiCallMetricFormatSchema$1>;
@@ -3888,8 +3888,8 @@ declare const llmCallMetricSchema: z.ZodObject<{
     string: "string";
     number: "number";
     boolean: "boolean";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -3917,8 +3917,8 @@ declare const apiCallMetricSchema: z.ZodObject<{
     string: "string";
     number: "number";
     boolean: "boolean";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -4031,8 +4031,8 @@ declare const llmCallsConfigSchema: z.ZodObject<{
       string: "string";
       number: "number";
       boolean: "boolean";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -4067,8 +4067,8 @@ declare const apiCallsConfigSchema: z.ZodObject<{
       string: "string";
       number: "number";
       boolean: "boolean";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z.ZodOptional<z.ZodType<NumberDisplayOptions, unknown, z.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -4567,9 +4567,9 @@ declare function extractApiCalls(spans: EvalTraceSpan$1[], config: ResolvedApiCa
  * - `refresh`: never read, always write (forces re-execution and overwrites).
  */
 declare const cacheModeSchema: z.ZodEnum<{
-  refresh: "refresh";
-  bypass: "bypass";
   use: "use";
+  bypass: "bypass";
+  refresh: "refresh";
 }>;
 /** Mode controlling how cached spans behave during a run. */
 type CacheMode = z.infer<typeof cacheModeSchema>;
@@ -4583,17 +4583,17 @@ declare const spanCacheOptionsSchema: z.ZodObject<{
 type SpanCacheOptions = z.infer<typeof spanCacheOptionsSchema>;
 /** Category of operation stored in the eval cache. */
 declare const cacheOperationTypeSchema: z.ZodEnum<{
-  value: "value";
   span: "span";
+  value: "value";
 }>;
 /** Category of operation stored in the eval cache. */
 type CacheOperationType = z.infer<typeof cacheOperationTypeSchema>;
 /** Status of a cache lookup recorded on a span or case scope. */
 declare const cacheStatusSchema: z.ZodEnum<{
+  bypass: "bypass";
+  refresh: "refresh";
   hit: "hit";
   miss: "miss";
-  refresh: "refresh";
-  bypass: "bypass";
 }>;
 /** Status of a cache lookup recorded on a span or case scope. */
 type CacheStatus = z.infer<typeof cacheStatusSchema>;
@@ -4610,10 +4610,10 @@ declare const traceCacheRefSchema: z.ZodObject<{
   namespace: z.ZodString;
   key: z.ZodString;
   status: z.ZodEnum<{
+    bypass: "bypass";
+    refresh: "refresh";
     hit: "hit";
     miss: "miss";
-    refresh: "refresh";
-    bypass: "bypass";
   }>;
   read: z.ZodOptional<z.ZodBoolean>;
   stored: z.ZodOptional<z.ZodBoolean>;
@@ -4670,7 +4670,6 @@ declare const cacheRecordingOpSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
@@ -4678,6 +4677,7 @@ declare const cacheRecordingOpSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
       pdf: "pdf";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -4721,8 +4721,8 @@ declare const cacheRecordingSchema: z.ZodObject<{
   finalStatus: z.ZodOptional<z.ZodEnum<{
     error: "error";
     running: "running";
-    cancelled: "cancelled";
     ok: "ok";
+    cancelled: "cancelled";
   }>>;
   finalError: z.ZodOptional<z.ZodObject<{
     name: z.ZodOptional<z.ZodString>;
@@ -4758,7 +4758,6 @@ declare const cacheRecordingSchema: z.ZodObject<{
         number: "number";
         boolean: "boolean";
         file: "file";
-        duration: "duration";
         markdown: "markdown";
         json: "json";
         image: "image";
@@ -4766,6 +4765,7 @@ declare const cacheRecordingSchema: z.ZodObject<{
         pdf: "pdf";
         audio: "audio";
         video: "video";
+        duration: "duration";
         percent: "percent";
         passFail: "passFail";
         stars: "stars";
@@ -4809,8 +4809,8 @@ declare const cacheEntrySchema: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodOptional<z.ZodEnum<{
-    value: "value";
     span: "span";
+    value: "value";
   }>>;
   operationName: z.ZodOptional<z.ZodString>;
   spanName: z.ZodOptional<z.ZodString>;
@@ -4822,8 +4822,8 @@ declare const cacheEntrySchema: z.ZodObject<{
     finalStatus: z.ZodOptional<z.ZodEnum<{
       error: "error";
       running: "running";
-      cancelled: "cancelled";
       ok: "ok";
+      cancelled: "cancelled";
     }>>;
     finalError: z.ZodOptional<z.ZodObject<{
       name: z.ZodOptional<z.ZodString>;
@@ -4859,7 +4859,6 @@ declare const cacheEntrySchema: z.ZodObject<{
           number: "number";
           boolean: "boolean";
           file: "file";
-          duration: "duration";
           markdown: "markdown";
           json: "json";
           image: "image";
@@ -4867,6 +4866,7 @@ declare const cacheEntrySchema: z.ZodObject<{
           pdf: "pdf";
           audio: "audio";
           video: "video";
+          duration: "duration";
           percent: "percent";
           passFail: "passFail";
           stars: "stars";
@@ -4916,8 +4916,8 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodEnum<{
-    value: "value";
     span: "span";
+    value: "value";
   }>;
   operationName: z.ZodString;
   storedAt: z.ZodString;
@@ -4927,8 +4927,8 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodOptional<z.ZodEnum<{
-      value: "value";
       span: "span";
+      value: "value";
     }>>;
     operationName: z.ZodOptional<z.ZodString>;
     spanName: z.ZodOptional<z.ZodString>;
@@ -4940,8 +4940,8 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
       finalStatus: z.ZodOptional<z.ZodEnum<{
         error: "error";
         running: "running";
-        cancelled: "cancelled";
         ok: "ok";
+        cancelled: "cancelled";
       }>>;
       finalError: z.ZodOptional<z.ZodObject<{
         name: z.ZodOptional<z.ZodString>;
@@ -4977,7 +4977,6 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
             number: "number";
             boolean: "boolean";
             file: "file";
-            duration: "duration";
             markdown: "markdown";
             json: "json";
             image: "image";
@@ -4985,6 +4984,7 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
             pdf: "pdf";
             audio: "audio";
             video: "video";
+            duration: "duration";
             percent: "percent";
             passFail: "passFail";
             stars: "stars";
@@ -5034,8 +5034,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodOptional<z.ZodEnum<{
-    value: "value";
     span: "span";
+    value: "value";
   }>>;
   operationName: z.ZodOptional<z.ZodString>;
   spanName: z.ZodOptional<z.ZodString>;
@@ -5047,8 +5047,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
     finalStatus: z.ZodOptional<z.ZodEnum<{
       error: "error";
       running: "running";
-      cancelled: "cancelled";
       ok: "ok";
+      cancelled: "cancelled";
     }>>;
     finalError: z.ZodOptional<z.ZodObject<{
       name: z.ZodOptional<z.ZodString>;
@@ -5084,7 +5084,6 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
           number: "number";
           boolean: "boolean";
           file: "file";
-          duration: "duration";
           markdown: "markdown";
           json: "json";
           image: "image";
@@ -5092,6 +5091,7 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
           pdf: "pdf";
           audio: "audio";
           video: "video";
+          duration: "duration";
           percent: "percent";
           passFail: "passFail";
           stars: "stars";
@@ -5132,8 +5132,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodEnum<{
-      value: "value";
       span: "span";
+      value: "value";
     }>;
     operationName: z.ZodString;
     storedAt: z.ZodString;
@@ -5143,8 +5143,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
       key: z.ZodString;
       namespace: z.ZodString;
       operationType: z.ZodOptional<z.ZodEnum<{
-        value: "value";
         span: "span";
+        value: "value";
       }>>;
       operationName: z.ZodOptional<z.ZodString>;
       spanName: z.ZodOptional<z.ZodString>;
@@ -5156,8 +5156,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
         finalStatus: z.ZodOptional<z.ZodEnum<{
           error: "error";
           running: "running";
-          cancelled: "cancelled";
           ok: "ok";
+          cancelled: "cancelled";
         }>>;
         finalError: z.ZodOptional<z.ZodObject<{
           name: z.ZodOptional<z.ZodString>;
@@ -5193,7 +5193,6 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
               number: "number";
               boolean: "boolean";
               file: "file";
-              duration: "duration";
               markdown: "markdown";
               json: "json";
               image: "image";
@@ -5201,6 +5200,7 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
               pdf: "pdf";
               audio: "audio";
               video: "video";
+              duration: "duration";
               percent: "percent";
               passFail: "passFail";
               stars: "stars";
@@ -5250,8 +5250,8 @@ declare const cacheFileSchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodOptional<z.ZodEnum<{
-      value: "value";
       span: "span";
+      value: "value";
     }>>;
     operationName: z.ZodOptional<z.ZodString>;
     spanName: z.ZodOptional<z.ZodString>;
@@ -5263,8 +5263,8 @@ declare const cacheFileSchema: z.ZodObject<{
       finalStatus: z.ZodOptional<z.ZodEnum<{
         error: "error";
         running: "running";
-        cancelled: "cancelled";
         ok: "ok";
+        cancelled: "cancelled";
       }>>;
       finalError: z.ZodOptional<z.ZodObject<{
         name: z.ZodOptional<z.ZodString>;
@@ -5300,7 +5300,6 @@ declare const cacheFileSchema: z.ZodObject<{
             number: "number";
             boolean: "boolean";
             file: "file";
-            duration: "duration";
             markdown: "markdown";
             json: "json";
             image: "image";
@@ -5308,6 +5307,7 @@ declare const cacheFileSchema: z.ZodObject<{
             pdf: "pdf";
             audio: "audio";
             video: "video";
+            duration: "duration";
             percent: "percent";
             passFail: "passFail";
             stars: "stars";
@@ -5356,8 +5356,8 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodEnum<{
-      value: "value";
       span: "span";
+      value: "value";
     }>;
     operationName: z.ZodString;
     storedAt: z.ZodString;
@@ -5367,8 +5367,8 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
       key: z.ZodString;
       namespace: z.ZodString;
       operationType: z.ZodOptional<z.ZodEnum<{
-        value: "value";
         span: "span";
+        value: "value";
       }>>;
       operationName: z.ZodOptional<z.ZodString>;
       spanName: z.ZodOptional<z.ZodString>;
@@ -5380,8 +5380,8 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
         finalStatus: z.ZodOptional<z.ZodEnum<{
           error: "error";
           running: "running";
-          cancelled: "cancelled";
           ok: "ok";
+          cancelled: "cancelled";
         }>>;
         finalError: z.ZodOptional<z.ZodObject<{
           name: z.ZodOptional<z.ZodString>;
@@ -5417,7 +5417,6 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
               number: "number";
               boolean: "boolean";
               file: "file";
-              duration: "duration";
               markdown: "markdown";
               json: "json";
               image: "image";
@@ -5425,6 +5424,7 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
               pdf: "pdf";
               audio: "audio";
               video: "video";
+              duration: "duration";
               percent: "percent";
               passFail: "passFail";
               stars: "stars";
@@ -5579,9 +5579,9 @@ declare const createRunRequestSchema$1: z.ZodObject<{
   temporary: z.ZodOptional<z.ZodBoolean>;
   cache: z.ZodOptional<z.ZodObject<{
     mode: z.ZodDefault<z.ZodEnum<{
-      refresh: "refresh";
-      bypass: "bypass";
       use: "use";
+      bypass: "bypass";
+      refresh: "refresh";
     }>>;
   }, z.core.$strip>>;
   manualInputs: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as setScopeCacheContext, A as repoFile, B as evalTime, C as evalTracer, D as deserializeCacheValue, E as deserializeCacheRecording, F as EvalRuntimeUsageError, Ft as getEvalRegistry, H as getEvalCaseInput, I as appendToEvalOutput, J as runInEvalRuntimeScope, K as mergeEvalOutput, M as readManualInputFile, N as evalExpect, O as serializeCacheRecording, P as EvalAssertionError, Q as setEvalOutput, R as evalAssert, S as evalSpan, T as hashCacheKeySync, U as incrementEvalOutput, V as getCurrentScope, W as isInEvalScope, X as runInExistingEvalScope, Y as runInEvalScope, at as extractApiCalls, b as buildTraceTree, ct as simulateTokenAllocation, et as startEvalBackgroundJob, it as extractCacheHits, j as manualInputFileValueSchema, k as serializeCacheValue, ot as extractLlmCalls, q as nextEvalId, rt as extractCacheEntries, st as simulateLlmCallCost, ut as getNestedAttribute, w as hashCacheKey, x as captureEvalSpanError, z as evalLog } from "./runExecution-Bq0Y3y_1.mjs";
-import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-CbePEEua.mjs";
-import { n as matchesEvalTags, t as defineEval } from "./src-CVM_FqPx.mjs";
+import { $ as startEvalBackgroundJob, A as manualInputFileValueSchema, B as getCurrentScope, C as hashCacheKey, D as serializeCacheRecording, E as deserializeCacheValue, F as appendToEvalOutput, G as mergeEvalOutput, H as incrementEvalOutput, J as runInEvalScope, K as nextEvalId, L as evalAssert, M as evalExpect, N as EvalAssertionError, O as serializeCacheValue, P as EvalRuntimeUsageError, Pt as getEvalRegistry, Q as setScopeCacheContext, R as evalLog, S as evalTracer, T as deserializeCacheRecording, U as isInEvalScope, V as getEvalCaseInput, Y as runInExistingEvalScope, Z as setEvalOutput, at as extractLlmCalls, b as captureEvalSpanError, it as extractApiCalls, j as readManualInputFile, k as repoFile, lt as getNestedAttribute, nt as extractCacheEntries, ot as simulateLlmCallCost, q as runInEvalRuntimeScope, rt as extractCacheHits, st as simulateTokenAllocation, w as hashCacheKeySync, x as evalSpan, y as buildTraceTree, z as evalTime } from "./runExecution-Bu9yfdUS.mjs";
+import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-OLZIjQpx.mjs";
+import { n as matchesEvalTags, t as defineEval } from "./src-Cy3OxoZW.mjs";
 export { EvalAssertionError, EvalRuntimeUsageError, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob };

package/dist/runChild.mjs

@@ -1,5 +1,5 @@
-import { At as evalStatsConfigSchema, L as configureEvalRunLogs, Mt as evalChartsConfigSchema, Nt as columnDefSchema, Tt as buildEvalKey, bt as runManifestSchema, jt as manualInputDescriptorSchema, kt as evalStatAggregateSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, tt as createRunRequestSchema, v as createFsCacheStore, xt as runSummarySchema, y as getCacheRetentionOptions } from "./runExecution-Bq0Y3y_1.mjs";
-import { S as parseEvalDiscovery, m as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-BpwW0AmB.mjs";
+import { At as manualInputDescriptorSchema, I as configureEvalRunLogs, Mt as columnDefSchema, Ot as evalStatAggregateSchema, _ as createFsCacheStore, bt as runSummarySchema, et as createRunRequestSchema, jt as evalChartsConfigSchema, kt as evalStatsConfigSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, v as getCacheRetentionOptions, wt as buildEvalKey, yt as runManifestSchema } from "./runExecution-Bu9yfdUS.mjs";
+import { C as parseEvalDiscovery, h as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-mpgZmEZ6.mjs";
 import { z } from "zod";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";

package/dist/{runExecution-Bq0Y3y_1.mjs → runExecution-Bu9yfdUS.mjs}

@@ -7239,4 +7239,4 @@ function recordAssertionFailure(scope, failure) {
 	});
 }
 //#endregion
-export { setScopeCacheContext as $, repoFile as A, evalStatsConfigSchema as At, evalTime as B, evalTracer as C, resolveLlmCallsConfig as Ct, deserializeCacheValue as D, caseDetailSchema as Dt, deserializeCacheRecording as E, getCaseRowCaseKey as Et, EvalRuntimeUsageError as F, getEvalRegistry as Ft, matchesEvalTags as G, getEvalCaseInput as H, appendToEvalOutput as I, runWithEvalRegistry as It, runInEvalRuntimeScope as J, mergeEvalOutput as K, configureEvalRunLogs as L, readManualInputFile as M, evalChartsConfigSchema as Mt, evalExpect as N, columnDefSchema as Nt, serializeCacheRecording as O, caseRowSchema as Ot, EvalAssertionError as P, defineEval as Pt, setEvalOutput as Q, evalAssert as R, evalSpan as S, resolveApiCallsConfig as St, hashCacheKeySync as T, buildEvalKey as Tt, incrementEvalOutput as U, getCurrentScope as V, isInEvalScope as W, runInExistingEvalScope as X, runInEvalScope as Y, runWithEvalClock as Z, createBufferedCacheStore as _, matchesTagsFilter as _t, isCaseChildParentMessage as a, extractApiCalls as at, buildTraceTree as b, runManifestSchema as bt, resolveArtifactPath as c, simulateTokenAllocation as ct, loadEvalModule as d, getEvalTitle as dt, startEvalBackgroundJob as et, resolveEvalDefaultConfig as f, getEvalDisplayStatus as ft, commitPendingCacheWrites as g, dedupeEvalTags as gt, normalizeScoreDef as h, deriveStatusFromChildStatuses as ht, isCaseChildMessage as i, extractCacheHits as it, manualInputFileValueSchema as j, manualInputDescriptorSchema as jt, serializeCacheValue as k, evalStatAggregateSchema as kt, registerAgentEvalsPackageResolutionHooks as l, applyDerivedCallAttributes as lt, buildDeclaredColumnDefs as m, deriveStatusFromCaseRows as mt, resolveRunnableEvalCases as n, updateManualScoreRequestSchema as nt, stripTerminalControlCodes as o, extractLlmCalls as ot, loadConfig as p, deriveScopedSummaryFromCases as pt, nextEvalId as q, runCase as r, extractCacheEntries as rt, resolveTracePresentation as s, simulateLlmCallCost as st, filterEvalCases as t, createRunRequestSchema as tt, runWithModuleIsolation as u, getNestedAttribute as ut, createFsCacheStore as v, validateEvalTagName as vt, hashCacheKey as w, buildCaseKey as wt, captureEvalSpanError as x, runSummarySchema as xt, getCacheRetentionOptions as y, validateTagsFilterExpression as yt, evalLog as z };
+export { startEvalBackgroundJob as $, manualInputFileValueSchema as A, manualInputDescriptorSchema as At, getCurrentScope as B, hashCacheKey as C, buildCaseKey as Ct, serializeCacheRecording as D, caseRowSchema as Dt, deserializeCacheValue as E, caseDetailSchema as Et, appendToEvalOutput as F, runWithEvalRegistry as Ft, mergeEvalOutput as G, incrementEvalOutput as H, configureEvalRunLogs as I, runInEvalScope as J, nextEvalId as K, evalAssert as L, evalExpect as M, columnDefSchema as Mt, EvalAssertionError as N, defineEval as Nt, serializeCacheValue as O, evalStatAggregateSchema as Ot, EvalRuntimeUsageError as P, getEvalRegistry as Pt, setScopeCacheContext as Q, evalLog as R, evalTracer as S, resolveLlmCallsConfig as St, deserializeCacheRecording as T, getCaseRowCaseKey as Tt, isInEvalScope as U, getEvalCaseInput as V, matchesEvalTags as W, runWithEvalClock as X, runInExistingEvalScope as Y, setEvalOutput as Z, createFsCacheStore as _, validateEvalTagName as _t, isCaseChildParentMessage as a, extractLlmCalls as at, captureEvalSpanError as b, runSummarySchema as bt, resolveArtifactPath as c, applyDerivedCallAttributes as ct, loadEvalModule as d, getEvalDisplayStatus as dt, createRunRequestSchema as et, resolveEvalDefaultConfig as f, deriveScopedSummaryFromCases as ft, createBufferedCacheStore as g, matchesTagsFilter as gt, commitPendingCacheWrites as h, dedupeEvalTags as ht, isCaseChildMessage as i, extractApiCalls as it, readManualInputFile as j, evalChartsConfigSchema as jt, repoFile as k, evalStatsConfigSchema as kt, registerAgentEvalsPackageResolutionHooks as l, getNestedAttribute as lt, buildDeclaredColumnDefs as m, deriveStatusFromChildStatuses as mt, resolveRunnableEvalCases as n, extractCacheEntries as nt, stripTerminalControlCodes as o, simulateLlmCallCost as ot, loadConfig as p, deriveStatusFromCaseRows as pt, runInEvalRuntimeScope as q, runCase as r, extractCacheHits as rt, resolveTracePresentation as s, simulateTokenAllocation as st, filterEvalCases as t, updateManualScoreRequestSchema as tt, runWithModuleIsolation as u, getEvalTitle as ut, getCacheRetentionOptions as v, validateTagsFilterExpression as vt, hashCacheKeySync as w, buildEvalKey as wt, evalSpan as x, resolveApiCallsConfig as xt, buildTraceTree as y, runManifestSchema as yt, evalTime as z };

package/dist/{runOrchestration-BpwW0AmB.mjs → runOrchestration-mpgZmEZ6.mjs}

@@ -1,8 +1,8 @@
-import { Dt as caseDetailSchema, Et as getCaseRowCaseKey, It as runWithEvalRegistry, J as runInEvalRuntimeScope, Ot as caseRowSchema, Z as runWithEvalClock, _t as matchesTagsFilter, bt as runManifestSchema, d as loadEvalModule, f as resolveEvalDefaultConfig, g as commitPendingCacheWrites, gt as dedupeEvalTags, ht as deriveStatusFromChildStatuses, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromCaseRows, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveScopedSummaryFromCases, t as filterEvalCases, u as runWithModuleIsolation, vt as validateEvalTagName, wt as buildCaseKey, xt as runSummarySchema, yt as validateTagsFilterExpression } from "./runExecution-Bq0Y3y_1.mjs";
+import { Ct as buildCaseKey, Dt as caseRowSchema, Et as caseDetailSchema, Ft as runWithEvalRegistry, Tt as getCaseRowCaseKey, X as runWithEvalClock, _t as validateEvalTagName, bt as runSummarySchema, d as loadEvalModule, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as commitPendingCacheWrites, ht as dedupeEvalTags, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromChildStatuses, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveStatusFromCaseRows, q as runInEvalRuntimeScope, t as filterEvalCases, u as runWithModuleIsolation, vt as validateTagsFilterExpression, yt as runManifestSchema } from "./runExecution-Bu9yfdUS.mjs";
 import { Result, resultify } from "t-result";
 import { readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
 //#region ../runner/src/chartValidation.ts
@@ -670,7 +670,7 @@ async function recomputeEvalStatusesInRuns(params) {
 		let changed = false;
 		for (const caseRow of run.cases) {
 			if (caseRow.evalKey !== params.evalKey) continue;
-			const caseDetail = run.caseDetails.get(getCaseRowCaseKey(caseRow));
+			const caseDetail = params.getCaseDetail?.(run, caseRow) ?? run.caseDetails.get(getCaseRowCaseKey(caseRow));
 			const nextStatus = recomputePersistedCaseStatus(caseRow, caseDetail, params.scoreThresholds);
 			if (caseRow.status === nextStatus) continue;
 			caseRow.status = nextStatus;
@@ -725,14 +725,22 @@ function nextShortIdFromSnapshots(snapshots) {
 	}
 	return maxNum + 1;
 }
-async function loadPersistedRunSnapshots(localStateDir) {
+/**
+* Load persisted run metadata from the local state directory.
+*
+* Case details are skipped by default so long-running app processes can keep
+* run history in memory without retaining every trace payload. Pass
+* `includeCaseDetails` only for narrow maintenance flows that need full
+* details for every case.
+*/
+async function loadPersistedRunSnapshots(localStateDir, options = {}) {
 	const runsDir = join(localStateDir, "runs");
 	const entriesResult = await resultify(() => readdir(runsDir, { withFileTypes: true }));
 	if (entriesResult.error) return [];
 	const snapshots = [];
 	const runDirs = entriesResult.value.filter((entry) => entry.isDirectory()).map((entry) => join(runsDir, entry.name)).toSorted();
 	for (const runDir of runDirs) {
-		const snapshot = await loadPersistedRunSnapshot(runDir);
+		const snapshot = await loadPersistedRunSnapshot(runDir, options);
 		if (!snapshot) continue;
 		snapshots.push(snapshot);
 	}
@@ -766,7 +774,14 @@ function getLatestRunInfos(params) {
 function toLastRunStatus$1(status) {
 	return status === "pending" ? null : status;
 }
-async function loadPersistedRunSnapshot(runDir) {
+/**
+* Load one persisted run snapshot from disk.
+*
+* The returned snapshot includes manifest, summary, and case rows. Case
+* details are loaded only when `includeCaseDetails` is true; otherwise callers
+* should use `loadPersistedCaseDetail` for the specific case being inspected.
+*/
+async function loadPersistedRunSnapshot(runDir, options = {}) {
 	const manifest = await readParsedJsonFile(join(runDir, "run.json"), { safeParse: runManifestSchema.safeParse.bind(runManifestSchema) });
 	if (!manifest) return null;
 	const summary = await readParsedJsonFile(join(runDir, "summary.json"), { safeParse: runSummarySchema.safeParse.bind(runSummarySchema) });
@@ -776,9 +791,18 @@ async function loadPersistedRunSnapshot(runDir) {
 		manifest,
 		summary,
 		cases: await readCaseRows(runDir),
-		caseDetails: await readCaseDetails(runDir)
+		caseDetails: options.includeCaseDetails === true ? await readCaseDetails(runDir) : /* @__PURE__ */ new Map()
 	};
 }
+/**
+* Load one persisted case detail by its artifact file id.
+*
+* Returns `null` when the file is missing, invalid JSON, or no longer matches
+* the current case-detail schema.
+*/
+function loadPersistedCaseDetail(runDir, fileId) {
+	return readParsedJsonFileSync(join(runDir, "case-details", `${encodeCaseDetailFileName(fileId)}.json`), { safeParse: caseDetailSchema.safeParse.bind(caseDetailSchema) });
+}
 async function readParsedJsonFile(filePath, schema) {
 	const fileResult = await resultify(() => readFile(filePath, "utf-8"));
 	if (fileResult.error) return null;
@@ -788,6 +812,15 @@ async function readParsedJsonFile(filePath, schema) {
 	if (!parsed.success) return null;
 	return parsed.data;
 }
+function readParsedJsonFileSync(filePath, schema) {
+	const fileResult = resultify(() => readFileSync(filePath, "utf-8"));
+	if (fileResult.error) return null;
+	const jsonResult = resultify(() => JSON.parse(fileResult.value));
+	if (jsonResult.error) return null;
+	const parsed = schema.safeParse(jsonResult.value);
+	if (!parsed.success) return null;
+	return parsed.data;
+}
 async function readCaseRows(runDir) {
 	const fileResult = await resultify(() => readFile(join(runDir, "cases.jsonl"), "utf-8"));
 	if (fileResult.error) return [];
@@ -1660,4 +1693,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { validateCharts as C, parseEvalDiscovery as S, runTouchesEval as _, validateTagsFilters as a, deriveEvalFreshness as b, getLatestRunInfos as c, nextShortIdFromSnapshots as d, persistCaseDetail as f, recomputePersistedCaseStatus as g, recomputeEvalStatusesInRuns as h, resolveEvalTags as i, loadPersistedRunSnapshot as l, persistRunState as m, getTargetEvalKeys as n, generateRunId as o, deleteTemporaryRuns as p, getTargetEvals as r, getLastRunStatuses as s, executeRun as t, loadPersistedRunSnapshots as u, buildManualInputDescriptor as v, loadIsolatedEvalRegistry as x, parseManualInputValues as y };
+export { parseEvalDiscovery as C, loadIsolatedEvalRegistry as S, recomputePersistedCaseStatus as _, validateTagsFilters as a, parseManualInputValues as b, getLatestRunInfos as c, loadPersistedRunSnapshots as d, nextShortIdFromSnapshots as f, recomputeEvalStatusesInRuns as g, persistRunState as h, resolveEvalTags as i, loadPersistedCaseDetail as l, deleteTemporaryRuns as m, getTargetEvalKeys as n, generateRunId as o, persistCaseDetail as p, getTargetEvals as r, getLastRunStatuses as s, executeRun as t, loadPersistedRunSnapshot as u, runTouchesEval as v, validateCharts as w, deriveEvalFreshness as x, buildManualInputDescriptor as y };

package/dist/{runner-XEP21_u9.mjs → runner-C4Y0lWb1.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-Kp0JqxrU.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-SxtKn-Xh.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-Kp0JqxrU.mjs → runner-SxtKn-Xh.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-CbePEEua.mjs";
-import "./src-CVM_FqPx.mjs";
+import { n as createRunner } from "./cli-OLZIjQpx.mjs";
+import "./src-Cy3OxoZW.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance({ loadEnv = true } = {}) {

package/dist/{src-CVM_FqPx.mjs → src-Cy3OxoZW.mjs}

@@ -1,5 +1,5 @@
-import { G as matchesEvalTags$1, Pt as defineEval$1 } from "./runExecution-Bq0Y3y_1.mjs";
-import "./cli-CbePEEua.mjs";
+import { Nt as defineEval$1, W as matchesEvalTags$1 } from "./runExecution-Bu9yfdUS.mjs";
+import "./cli-OLZIjQpx.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.60.2",
+  "version": "0.60.4",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -32,9 +32,9 @@
   "devDependencies": {
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
-    "@agent-evals/runner": "0.0.1",
     "@agent-evals/shared": "0.0.1",
-    "@agent-evals/sdk": "0.0.1"
+    "@agent-evals/sdk": "0.0.1",
+    "@agent-evals/runner": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",