@ls-stack/agent-eval 0.36.0 → 0.37.0

package/dist/{cli-Dg3abrOv.mjs → cli-CwGcJYWe.mjs}

@@ -1,4 +1,4 @@
-import { C as loadConfig, D as createFsCacheStore, E as validateCharts, H as getEvalDisplayStatus, S as resolveEvalDefaultConfig, T as normalizeScoreDef, U as deriveScopedSummaryFromCases, V as getEvalTitle, _ as buildManualInputDescriptor, a as getLastRunStatuses, b as loadEvalModule, bt as getCaseRowEvalKey, c as loadPersistedRunSnapshots, d as persistRunState, f as recomputeEvalStatusesInRuns, g as resolveArtifactPath, h as resolveTracePresentation, i as generateRunId, l as nextShortIdFromSnapshots, m as runTouchesEval, mt as resolveLlmCallsConfig, n as getTargetEvalKeys, o as getLatestRunInfos, p as recomputePersistedCaseStatus, pr as getEvalRegistry, pt as resolveApiCallsConfig, q as runSummarySchema, s as loadPersistedRunSnapshot, u as persistCaseDetail, v as parseManualInputValues, vt as buildEvalKey, w as buildDeclaredColumnDefs, x as parseEvalDiscovery, y as deriveEvalFreshness, yt as getCaseRowCaseKey, z as applyDerivedCallAttributes } from "./runOrchestration-V1TxX8es.mjs";
+import { C as parseEvalDiscovery, D as normalizeScoreDef, E as buildDeclaredColumnDefs, G as deriveScopedSummaryFromCases, O as validateCharts, S as loadEvalModule, St as getCaseRowEvalKey, T as loadConfig, U as getEvalTitle, V as applyDerivedCallAttributes, W as getEvalDisplayStatus, Y as runSummarySchema, _ as resolveTracePresentation, a as generateRunId, b as parseManualInputValues, bt as buildEvalKey, c as loadPersistedRunSnapshot, d as persistCaseDetail, f as deleteTemporaryRuns, g as runTouchesEval, gt as resolveLlmCallsConfig, h as recomputePersistedCaseStatus, hr as getEvalRegistry, ht as resolveApiCallsConfig, i as stripTerminalControlCodes, k as createFsCacheStore, l as loadPersistedRunSnapshots, m as recomputeEvalStatusesInRuns, n as getTargetEvalKeys, o as getLastRunStatuses, p as persistRunState, s as getLatestRunInfos, u as nextShortIdFromSnapshots, v as resolveArtifactPath, w as resolveEvalDefaultConfig, x as deriveEvalFreshness, xt as getCaseRowCaseKey, y as buildManualInputDescriptor } from "./runOrchestration-C4o5TcIu.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -536,6 +536,7 @@ function isRunChildMessage(value) {
 const runChildInspectArgEnv = "AGENT_EVALS_RUN_CHILD_INSPECT_ARG";
 const inspectFlagPrefix = "--inspect";
 const inspectBrkFlagPrefix = "--inspect-brk";
+const childOutputTailMaxLength = 12e3;
 function startRunChild(params) {
 	const child = spawn(process.execPath, [
 		...getRunChildExecArgv(),
@@ -546,11 +547,12 @@ function startRunChild(params) {
 		env: process.env,
 		stdio: [
 			"ignore",
-			"inherit",
-			"inherit",
+			"pipe",
+			"pipe",
 			"ipc"
 		]
 	});
+	const outputTail = createRunChildOutputTail(child);
 	params.runState.childProcess = child;
 	child.on("message", (message) => {
 		if (!isRunChildMessage(message)) return;
@@ -564,8 +566,39 @@ function startRunChild(params) {
 		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
 		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
 		const reason = signal !== null ? `Run child exited with signal ${signal}` : `Run child exited with code ${String(code)}`;
-		markRunErrored(params.runState, reason, params.managerContext);
+		markRunErrored(params.runState, formatUnexpectedRunChildExit(reason, outputTail), params.managerContext);
+	});
+}
+function createRunChildOutputTail(child) {
+	const tail = {
+		stdout: "",
+		stderr: ""
+	};
+	child.stdout?.on("data", (chunk) => {
+		process.stdout.write(chunk);
+		tail.stdout = appendOutputTail(tail.stdout, chunkToText(chunk));
+	});
+	child.stderr?.on("data", (chunk) => {
+		process.stderr.write(chunk);
+		tail.stderr = appendOutputTail(tail.stderr, chunkToText(chunk));
 	});
+	return tail;
+}
+function chunkToText(chunk) {
+	return typeof chunk === "string" ? chunk : chunk.toString("utf-8");
+}
+function appendOutputTail(current, next) {
+	const combined = current + next;
+	if (combined.length <= childOutputTailMaxLength) return combined;
+	return combined.slice(combined.length - childOutputTailMaxLength);
+}
+function formatUnexpectedRunChildExit(reason, outputTail) {
+	const sections = [reason];
+	const stderr = stripTerminalControlCodes(outputTail.stderr).trim();
+	const stdout = stripTerminalControlCodes(outputTail.stdout).trim();
+	if (stderr.length > 0) sections.push(`Child stderr (last ${String(stderr.length)} chars):\n${stderr}`);
+	if (stdout.length > 0) sections.push(`Child stdout (last ${String(stdout.length)} chars):\n${stdout}`);
+	return sections.join("\n\n");
 }
 function getRunChildExecArgv() {
 	const execArgv = [];
@@ -647,6 +680,7 @@ function applyChildEvalMetas(evals, childMetas) {
 		}
 		evalMeta.columnDefs = childMeta.columnDefs;
 		evalMeta.caseCount = childMeta.caseCount;
+		evalMeta.caseIds = childMeta.caseIds;
 		evalMeta.stats = childMeta.stats;
 		evalMeta.charts = childMeta.charts;
 		evalMeta.sourceFingerprint = childMeta.sourceFingerprint;
@@ -1064,6 +1098,7 @@ function createRunner({ watchForChanges = true } = {}) {
 						sourceFingerprint,
 						columnDefs,
 						caseCount: null,
+						caseIds: void 0,
 						stats,
 						charts,
 						manualInputDescriptor,
@@ -1074,18 +1109,24 @@ function createRunner({ watchForChanges = true } = {}) {
 			emitDiscoveryEvent();
 		},
 		async startRun(request) {
+			const deletedTemporaryRuns = await deleteTemporaryRuns({
+				runs,
+				cancelRunningRun: killRunChild
+			});
 			const runId = generateRunId();
 			const shortId = `r${String(nextShortIdNum++)}`;
 			const now = (/* @__PURE__ */ new Date()).toISOString();
 			const cacheMode = request.cache?.mode ?? "use";
 			const runDir = join(localStateDir, "runs", runId);
+			const gitState = readGitWorktreeState(workspaceRoot);
 			const manifest = {
 				id: runId,
 				shortId,
 				status: "running",
+				temporary: request.temporary === true,
 				startedAt: now,
 				endedAt: null,
-				commitSha: readGitWorktreeState(workspaceRoot).commitSha,
+				commitSha: gitState.commitSha,
 				evalSourceFingerprints: {},
 				target: request.target,
 				trials: request.trials,
@@ -1162,6 +1203,7 @@ function createRunner({ watchForChanges = true } = {}) {
 					emitDiscoveryEvent
 				}
 			});
+			if (deletedTemporaryRuns > 0) emitDiscoveryEvent();
 			return {
 				manifest,
 				summary,
@@ -1445,6 +1487,7 @@ Flags:
   --no-cache                 Shortcut for --cache bypass
   --refresh-cache            Shortcut for --cache refresh
   --clear-cache              Clear the cache before starting the run
+  --temporary                Persist until the next run starts, then delete
   --input <json>             Manual input value for a single targeted eval
                              that declares manualInput
   --input-file <path>        JSON object keyed by eval key (or eval id) with
@@ -1515,6 +1558,7 @@ Options:
   --no-cache                 Shortcut for --cache bypass
   --refresh-cache            Shortcut for --cache refresh
   --clear-cache              Clear the cache before starting the run
+  --temporary                Persist until the next run starts, then delete
   --no-env                   Disable automatic .env loading
   --help, -h                 Show help
   `);
@@ -1753,6 +1797,7 @@ function parseArgs(argv) {
 		port: 4100,
 		cacheMode: "use",
 		clearCache: false,
+		temporary: false,
 		all: false,
 		loadEnv: normalizedArgv.length === argv.length,
 		inputJson: void 0,
@@ -1803,6 +1848,7 @@ function parseArgs(argv) {
 		} else if (arg === "--no-cache") args.cacheMode = "bypass";
 		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
 		else if (arg === "--clear-cache") args.clearCache = true;
+		else if (arg === "--temporary") args.temporary = true;
 		else if (arg === "--input" && next !== void 0) {
 			args.inputJson = next;
 			i++;
@@ -1940,8 +1986,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-BlNzXWDM.mjs");
-	const runnerModule = await import("./runner-znY6PY1M.mjs");
+	const appModule = await import("./app-C7ON9Wdh.mjs");
+	const runnerModule = await import("./runner-LqeHPID6.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -2030,6 +2076,7 @@ async function commandRun(args) {
 	const run = await runner.startRun({
 		target,
 		trials: args.trials,
+		temporary: args.temporary,
 		cache: { mode: args.cacheMode },
 		manualInputs: manualInputsResult.value
 	});
@@ -2037,6 +2084,7 @@ async function commandRun(args) {
 		console.info(`Run started: ${run.manifest.id}`);
 		console.info(`Trials: ${String(args.trials)}`);
 		if (args.cacheMode !== "use") console.info(`Cache mode: ${args.cacheMode}`);
+		if (args.temporary) console.info("Temporary: yes");
 		console.info("");
 	}
 	await waitForRunCompletion(runner, run.manifest.id);
@@ -2152,6 +2200,7 @@ function buildRunFileIndex(workspaceRoot, run) {
 		id: run.manifest.id,
 		shortId: run.manifest.shortId,
 		status: run.manifest.status,
+		temporary: run.manifest.temporary,
 		startedAt: run.manifest.startedAt,
 		endedAt: run.manifest.endedAt,
 		target: run.manifest.target,
@@ -2212,7 +2261,7 @@ function printRunFileIndexes(indexes) {
 	}
 }
 function printRunFileIndex(index) {
-	console.info(`${index.shortId} (${index.id})  ${index.status}  ${formatCaseCounts(index.summary)}`);
+	console.info(`${index.shortId} (${index.id})  ${index.status}${index.temporary ? "  temporary" : ""}  ${formatCaseCounts(index.summary)}`);
 	console.info(`  dir: ${index.files.dir}`);
 	console.info(`  run: ${index.files.run}`);
 	console.info(`  summary: ${index.files.summary}`);

package/dist/index.d.mts

@@ -127,6 +127,7 @@ declare const evalSummarySchema$1: z$1.ZodObject<{
     }>>;
   }, z$1.core.$strip>>;
   caseCount: z$1.ZodNullable<z$1.ZodNumber>;
+  caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   lastRunStatus: z$1.ZodNullable<z$1.ZodEnum<{
     error: "error";
     pass: "pass";
@@ -617,6 +618,7 @@ declare const runManifestSchema$1: z$1.ZodObject<{
     cancelled: "cancelled";
     error: "error";
   }>;
+  temporary: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodBoolean>>;
   startedAt: z$1.ZodString;
   endedAt: z$1.ZodNullable<z$1.ZodString>;
   commitSha: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodNullable<z$1.ZodString>>>;
@@ -963,6 +965,7 @@ declare const createRunRequestSchema$1: z$1.ZodObject<{
     caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   }, z$1.core.$strip>;
   trials: z$1.ZodNumber;
+  temporary: z$1.ZodOptional<z$1.ZodBoolean>;
   cache: z$1.ZodOptional<z$1.ZodObject<{
     mode: z$1.ZodDefault<z$1.ZodEnum<{
       use: "use";
@@ -2916,12 +2919,12 @@ declare const columnFormatSchema: z$1.ZodEnum<{
   number: "number";
   boolean: "boolean";
   file: "file";
-  duration: "duration";
   markdown: "markdown";
   json: "json";
   image: "image";
   audio: "audio";
   video: "video";
+  duration: "duration";
   percent: "percent";
   passFail: "passFail";
   stars: "stars";
@@ -2941,12 +2944,12 @@ declare const columnDefSchema: z$1.ZodObject<{
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -2991,8 +2994,8 @@ declare const traceSpanKindSchema: z$1.ZodString;
 declare const traceAttributeDisplayFormatSchema: z$1.ZodEnum<{
   string: "string";
   number: "number";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /**
  * Formatting hint for trace attribute values rendered by the UI.
@@ -3016,8 +3019,8 @@ declare const traceAttributeDisplaySchema: z$1.ZodObject<{
   format: z$1.ZodOptional<z$1.ZodEnum<{
     string: "string";
     number: "number";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3052,8 +3055,8 @@ declare const traceDisplayConfigSchema: z$1.ZodObject<{
     format: z$1.ZodOptional<z$1.ZodEnum<{
       string: "string";
       number: "number";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3092,8 +3095,8 @@ declare const traceAttributeDisplayInputSchema: z$1.ZodObject<{
   format: z$1.ZodOptional<z$1.ZodEnum<{
     string: "string";
     number: "number";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3130,8 +3133,8 @@ declare const traceDisplayInputConfigSchema: z$1.ZodObject<{
     format: z$1.ZodOptional<z$1.ZodEnum<{
       string: "string";
       number: "number";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3263,12 +3266,12 @@ declare const evalStatItemSchema: z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -3305,12 +3308,12 @@ declare const evalStatsConfigSchema: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1
     number: "number";
     boolean: "boolean";
     file: "file";
-    duration: "duration";
     markdown: "markdown";
     json: "json";
     image: "image";
     audio: "audio";
     video: "video";
+    duration: "duration";
     percent: "percent";
     passFail: "passFail";
     stars: "stars";
@@ -3348,12 +3351,12 @@ declare const evalSummarySchema: z$1.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -3372,6 +3375,7 @@ declare const evalSummarySchema: z$1.ZodObject<{
     }>>;
   }, z$1.core.$strip>>;
   caseCount: z$1.ZodNullable<z$1.ZodNumber>;
+  caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   lastRunStatus: z$1.ZodNullable<z$1.ZodEnum<{
     error: "error";
     pass: "pass";
@@ -3406,12 +3410,12 @@ declare const evalSummarySchema: z$1.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -3437,8 +3441,8 @@ declare const evalSummarySchema: z$1.ZodObject<{
       label: z$1.ZodOptional<z$1.ZodString>;
       color: z$1.ZodOptional<z$1.ZodEnum<{
         success: "success";
-        accent: "accent";
         error: "error";
+        accent: "accent";
         accentDim: "accentDim";
         warning: "warning";
         textMuted: "textMuted";
@@ -3461,8 +3465,8 @@ declare const evalSummarySchema: z$1.ZodObject<{
       label: z$1.ZodOptional<z$1.ZodString>;
       color: z$1.ZodOptional<z$1.ZodEnum<{
         success: "success";
-        accent: "accent";
         error: "error";
+        accent: "accent";
         accentDim: "accentDim";
         warning: "warning";
         textMuted: "textMuted";
@@ -3725,8 +3729,8 @@ declare const scoreTraceSchema: z$1.ZodObject<{
       format: z$1.ZodOptional<z$1.ZodEnum<{
         string: "string";
         number: "number";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3811,8 +3815,8 @@ declare const caseDetailSchema: z$1.ZodObject<{
       format: z$1.ZodOptional<z$1.ZodEnum<{
         string: "string";
         number: "number";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -3880,8 +3884,8 @@ declare const caseDetailSchema: z$1.ZodObject<{
         format: z$1.ZodOptional<z$1.ZodEnum<{
           string: "string";
           number: "number";
-          duration: "duration";
           json: "json";
+          duration: "duration";
         }>>;
         numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
         placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -4045,8 +4049,8 @@ type EvalChartAggregate = z$1.infer<typeof evalChartAggregateSchema>;
  */
 declare const evalChartColorSchema: z$1.ZodEnum<{
   success: "success";
-  accent: "accent";
   error: "error";
+  accent: "accent";
   accentDim: "accentDim";
   warning: "warning";
   textMuted: "textMuted";
@@ -4074,8 +4078,8 @@ declare const evalChartMetricSchema: z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
   label: z$1.ZodOptional<z$1.ZodString>;
   color: z$1.ZodOptional<z$1.ZodEnum<{
     success: "success";
-    accent: "accent";
     error: "error";
+    accent: "accent";
     accentDim: "accentDim";
     warning: "warning";
     textMuted: "textMuted";
@@ -4098,8 +4102,8 @@ declare const evalChartMetricSchema: z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
   label: z$1.ZodOptional<z$1.ZodString>;
   color: z$1.ZodOptional<z$1.ZodEnum<{
     success: "success";
-    accent: "accent";
     error: "error";
+    accent: "accent";
     accentDim: "accentDim";
     warning: "warning";
     textMuted: "textMuted";
@@ -4157,8 +4161,8 @@ declare const evalChartConfigSchema: z$1.ZodObject<{
     label: z$1.ZodOptional<z$1.ZodString>;
     color: z$1.ZodOptional<z$1.ZodEnum<{
       success: "success";
-      accent: "accent";
       error: "error";
+      accent: "accent";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -4181,8 +4185,8 @@ declare const evalChartConfigSchema: z$1.ZodObject<{
     label: z$1.ZodOptional<z$1.ZodString>;
     color: z$1.ZodOptional<z$1.ZodEnum<{
       success: "success";
-      accent: "accent";
       error: "error";
+      accent: "accent";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -4247,8 +4251,8 @@ declare const evalChartsConfigSchema: z$1.ZodArray<z$1.ZodObject<{
     label: z$1.ZodOptional<z$1.ZodString>;
     color: z$1.ZodOptional<z$1.ZodEnum<{
       success: "success";
-      accent: "accent";
       error: "error";
+      accent: "accent";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -4271,8 +4275,8 @@ declare const evalChartsConfigSchema: z$1.ZodArray<z$1.ZodObject<{
     label: z$1.ZodOptional<z$1.ZodString>;
     color: z$1.ZodOptional<z$1.ZodEnum<{
       success: "success";
-      accent: "accent";
       error: "error";
+      accent: "accent";
       accentDim: "accentDim";
       warning: "warning";
       textMuted: "textMuted";
@@ -4327,15 +4331,16 @@ declare const runManifestSchema: z$1.ZodObject<{
     pending: "pending";
     completed: "completed";
   }>;
+  temporary: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodBoolean>>;
   startedAt: z$1.ZodString;
   endedAt: z$1.ZodNullable<z$1.ZodString>;
   commitSha: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodNullable<z$1.ZodString>>>;
   evalSourceFingerprints: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodRecord<z$1.ZodString, z$1.ZodString>>>;
   target: z$1.ZodObject<{
     mode: z$1.ZodEnum<{
+      caseIds: "caseIds";
       all: "all";
       evalIds: "evalIds";
-      caseIds: "caseIds";
     }>;
     evalKeys: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     files: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
@@ -4581,8 +4586,8 @@ declare const llmCallMetricFormatSchema: z$1.ZodEnum<{
   string: "string";
   number: "number";
   boolean: "boolean";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /** Render format applied to an LLM-call metric value. */
 type LlmCallMetricFormat = z$1.infer<typeof llmCallMetricFormatSchema>;
@@ -4591,8 +4596,8 @@ declare const apiCallMetricFormatSchema: z$1.ZodEnum<{
   string: "string";
   number: "number";
   boolean: "boolean";
-  duration: "duration";
   json: "json";
+  duration: "duration";
 }>;
 /** Render format applied to an API-call metric value. */
 type ApiCallMetricFormat = z$1.infer<typeof apiCallMetricFormatSchema>;
@@ -4661,8 +4666,8 @@ declare const llmCallMetricSchema: z$1.ZodObject<{
     string: "string";
     number: "number";
     boolean: "boolean";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -4690,8 +4695,8 @@ declare const apiCallMetricSchema: z$1.ZodObject<{
     string: "string";
     number: "number";
     boolean: "boolean";
-    duration: "duration";
     json: "json";
+    duration: "duration";
   }>>;
   numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
   placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -4804,8 +4809,8 @@ declare const llmCallsConfigSchema: z$1.ZodObject<{
       string: "string";
       number: "number";
       boolean: "boolean";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -4840,8 +4845,8 @@ declare const apiCallsConfigSchema: z$1.ZodObject<{
       string: "string";
       number: "number";
       boolean: "boolean";
-      duration: "duration";
       json: "json";
+      duration: "duration";
     }>>;
     numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
     placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -5142,8 +5147,8 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
       format: z$1.ZodOptional<z$1.ZodEnum<{
         string: "string";
         number: "number";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -5191,12 +5196,12 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
       number: "number";
       boolean: "boolean";
       file: "file";
-      duration: "duration";
       markdown: "markdown";
       json: "json";
       image: "image";
       audio: "audio";
       video: "video";
+      duration: "duration";
       percent: "percent";
       passFail: "passFail";
       stars: "stars";
@@ -5255,8 +5260,8 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
         string: "string";
         number: "number";
         boolean: "boolean";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -5300,8 +5305,8 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
         string: "string";
         number: "number";
         boolean: "boolean";
-        duration: "duration";
         json: "json";
+        duration: "duration";
       }>>;
       numberFormat: z$1.ZodOptional<z$1.ZodType<NumberDisplayOptions, unknown, z$1.core.$ZodTypeInternals<NumberDisplayOptions, unknown>>>;
       placements: z$1.ZodOptional<z$1.ZodArray<z$1.ZodEnum<{
@@ -6084,9 +6089,9 @@ type ConfigReloadState = z$1.infer<typeof configReloadStateSchema>;
 declare const createRunRequestSchema: z$1.ZodObject<{
   target: z$1.ZodObject<{
     mode: z$1.ZodEnum<{
+      caseIds: "caseIds";
       all: "all";
       evalIds: "evalIds";
-      caseIds: "caseIds";
     }>;
     evalKeys: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     files: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
@@ -6094,6 +6099,7 @@ declare const createRunRequestSchema: z$1.ZodObject<{
     caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   }, z$1.core.$strip>;
   trials: z$1.ZodNumber;
+  temporary: z$1.ZodOptional<z$1.ZodBoolean>;
   cache: z$1.ZodOptional<z$1.ZodObject<{
     mode: z$1.ZodDefault<z$1.ZodEnum<{
       refresh: "refresh";

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as apiCallMetricSchema, $n as getCurrentScope, $t as cacheDebugKeyEntrySchema, A as createRunRequestSchema, An as repoFileRefSchema, At as runLogEntrySchema, B as getNestedAttribute, Bn as deserializeCacheValue, Bt as manualInputNumberFieldSchema, Cn as cellValueSchema, Ct as caseRowSchema, Dn as fileRefSchema, Dt as evalStatItemSchema, En as columnKindSchema, Et as evalStatAggregateSchema, F as extractApiCalls, Fn as evalSpan, Ft as manualInputBooleanFieldSchema, G as deriveStatusFromChildStatuses, Gn as readManualInputFile, Gt as evalChartAxisSchema, H as getEvalDisplayStatus, Hn as serializeCacheValue, Ht as manualInputSelectOptionSchema, I as extractLlmCalls, In as evalTracer, It as manualInputDescriptorSchema, J as DEFAULT_API_CALLS_CONFIG, Jn as advanceEvalTime, Jt as evalChartConfigSchema, K as runManifestSchema, Kn as evalExpect, Kt as evalChartBuiltinMetricSchema, L as simulateLlmCallCost, Ln as hashCacheKey, Lt as manualInputFieldDescriptorSchema, M as sseEnvelopeSchema, Mn as z, Mt as runLogLocationSchema, N as extractCacheEntries, Nn as buildTraceTree, Nt as runLogPhaseSchema, O as configReloadStateSchema, On as jsonCellSchema, Ot as evalStatsConfigSchema, P as extractCacheHits, Pn as captureEvalSpanError, Pt as scoreTraceSchema, Q as apiCallMetricPlacementSchema, Qn as evalLog, Qt as evalChartsConfigSchema, R as simulateTokenAllocation, Rn as hashCacheKeySync, Rt as manualInputJsonFieldSchema, Sn as traceSpanWarningSchema, St as caseDetailSchema, Tn as columnFormatSchema, Tt as evalFreshnessStatusSchema, U as deriveScopedSummaryFromCases, Un as repoFile, Ut as manualInputTextFieldSchema, V as getEvalTitle, Vn as serializeCacheRecording, Vt as manualInputSelectFieldSchema, W as deriveStatusFromCaseRows, Wn as manualInputFileValueSchema, Wt as evalChartAggregateSchema, X as agentEvalsConfigSchema, Xt as evalChartTooltipExtraSchema, Y as DEFAULT_LLM_CALLS_CONFIG, Yn as appendToEvalOutput, Yt as evalChartMetricSchema, Z as apiCallMetricFormatSchema, Zn as evalAssert, Zt as evalChartTypeSchema, _n as traceDisplayConfigSchema, _t as buildCaseKey, an as cacheModeSchema, ar as nextEvalId, at as llmCallCostCurrencySchema, bn as traceSpanKindSchema, bt as getCaseRowEvalKey, cn as cacheRecordingSchema, cr as runInExistingEvalScope, ct as llmCallMetricSchema, dn as spanCacheOptionsSchema, dr as startEvalBackgroundJob, dt as llmCallsConfigSchema, en as cacheDebugKeyFileSchema, er as getEvalCaseInput, et as apiCallsConfigSchema, fn as traceCacheRefSchema, fr as defineEval, ft as removeDefaultConfigSchema, gn as traceAttributeDisplaySchema, gt as trialSelectionModeSchema, hn as traceAttributeDisplayPlacementSchema, ht as runLogsConfigSchema, in as cacheListItemSchema, ir as mergeEvalOutput, it as evalDeriveConfigSchema, j as updateManualScoreRequestSchema, jn as runArtifactRefSchema, jt as runLogLevelSchema, k as configReloadStatusSchema, kn as numberDisplayOptionsSchema, kt as evalSummarySchema, ln as cacheStatusSchema, lr as setEvalOutput, lt as llmCallPricingRateSchema, mn as traceAttributeDisplayInputSchema, mt as resolveLlmCallsConfig, nn as cacheEntryWithDebugKeySchema, nr as incrementEvalOutput, nt as evalColumnOverrideSchema, on as cacheOperationTypeSchema, or as runInEvalRuntimeScope, ot as llmCallMetricFormatSchema, pn as traceAttributeDisplayFormatSchema, pr as getEvalRegistry, pt as resolveApiCallsConfig, q as runSummarySchema, qn as EvalAssertionError, qt as evalChartColorSchema, rn as cacheFileSchema, rr as isInEvalScope, rt as evalColumnsSchema, sn as cacheRecordingOpSchema, sr as runInEvalScope, st as llmCallMetricPlacementSchema, tn as cacheEntrySchema, tr as getEvalStartTime, tt as defaultConfigKeySchema, un as serializedCacheSpanSchema, ur as setScopeCacheContext, ut as llmCallPricingSchema, vn as traceDisplayInputConfigSchema, vt as buildEvalKey, wn as columnDefSchema, wt as discoveryIssueSchema, xn as traceSpanSchema, xt as assertionFailureSchema, yn as traceSpanErrorSchema, yt as getCaseRowCaseKey, z as applyDerivedCallAttributes, zn as deserializeCacheRecording, zt as manualInputMultilineFieldSchema } from "./runOrchestration-V1TxX8es.mjs";
-import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-Dg3abrOv.mjs";
-import "./src-DBypR4TV.mjs";
+import { $ as apiCallMetricFormatSchema, $n as evalAssert, $t as evalChartTypeSchema, A as configReloadStateSchema, An as jsonCellSchema, At as evalStatsConfigSchema, B as simulateTokenAllocation, Bn as hashCacheKeySync, Bt as manualInputJsonFieldSchema, Cn as traceSpanSchema, Ct as assertionFailureSchema, Dn as columnFormatSchema, Dt as evalFreshnessStatusSchema, En as columnDefSchema, Et as discoveryIssueSchema, F as extractCacheEntries, Fn as buildTraceTree, Ft as runLogPhaseSchema, G as deriveScopedSummaryFromCases, Gn as repoFile, Gt as manualInputTextFieldSchema, H as getNestedAttribute, Hn as deserializeCacheValue, Ht as manualInputNumberFieldSchema, I as extractCacheHits, In as captureEvalSpanError, It as scoreTraceSchema, J as runManifestSchema, Jn as evalExpect, Jt as evalChartBuiltinMetricSchema, K as deriveStatusFromCaseRows, Kn as manualInputFileValueSchema, Kt as evalChartAggregateSchema, L as extractApiCalls, Ln as evalSpan, Lt as manualInputBooleanFieldSchema, M as createRunRequestSchema, Mn as repoFileRefSchema, Mt as runLogEntrySchema, N as updateManualScoreRequestSchema, Nn as runArtifactRefSchema, Nt as runLogLevelSchema, On as columnKindSchema, Ot as evalStatAggregateSchema, P as sseEnvelopeSchema, Pn as z, Pt as runLogLocationSchema, Q as agentEvalsConfigSchema, Qt as evalChartTooltipExtraSchema, R as extractLlmCalls, Rn as evalTracer, Rt as manualInputDescriptorSchema, Sn as traceSpanKindSchema, St as getCaseRowEvalKey, Tn as cellValueSchema, Tt as caseRowSchema, U as getEvalTitle, Un as serializeCacheRecording, Ut as manualInputSelectFieldSchema, V as applyDerivedCallAttributes, Vn as deserializeCacheRecording, Vt as manualInputMultilineFieldSchema, W as getEvalDisplayStatus, Wn as serializeCacheValue, Wt as manualInputSelectOptionSchema, X as DEFAULT_API_CALLS_CONFIG, Xn as advanceEvalTime, Xt as evalChartConfigSchema, Y as runSummarySchema, Yn as EvalAssertionError, Yt as evalChartColorSchema, Z as DEFAULT_LLM_CALLS_CONFIG, Zn as appendToEvalOutput, Zt as evalChartMetricSchema, _n as traceAttributeDisplayPlacementSchema, _t as runLogsConfigSchema, an as cacheFileSchema, ar as isInEvalScope, at as evalColumnsSchema, bn as traceDisplayInputConfigSchema, bt as buildEvalKey, cn as cacheOperationTypeSchema, cr as runInEvalRuntimeScope, ct as llmCallMetricFormatSchema, dn as cacheStatusSchema, dr as setEvalOutput, dt as llmCallPricingRateSchema, en as evalChartsConfigSchema, er as evalLog, et as apiCallMetricPlacementSchema, fn as serializedCacheSpanSchema, fr as setScopeCacheContext, ft as llmCallPricingSchema, gn as traceAttributeDisplayInputSchema, gt as resolveLlmCallsConfig, hn as traceAttributeDisplayFormatSchema, hr as getEvalRegistry, ht as resolveApiCallsConfig, in as cacheEntryWithDebugKeySchema, ir as incrementEvalOutput, it as evalColumnOverrideSchema, j as configReloadStatusSchema, jn as numberDisplayOptionsSchema, jt as evalSummarySchema, kn as fileRefSchema, kt as evalStatItemSchema, ln as cacheRecordingOpSchema, lr as runInEvalScope, lt as llmCallMetricPlacementSchema, mn as traceCacheRefSchema, mr as defineEval, mt as removeDefaultConfigSchema, nn as cacheDebugKeyFileSchema, nr as getEvalCaseInput, nt as apiCallsConfigSchema, on as cacheListItemSchema, or as mergeEvalOutput, ot as evalDeriveConfigSchema, pn as spanCacheOptionsSchema, pr as startEvalBackgroundJob, pt as llmCallsConfigSchema, q as deriveStatusFromChildStatuses, qn as readManualInputFile, qt as evalChartAxisSchema, rn as cacheEntrySchema, rr as getEvalStartTime, rt as defaultConfigKeySchema, sn as cacheModeSchema, sr as nextEvalId, st as llmCallCostCurrencySchema, tn as cacheDebugKeyEntrySchema, tr as getCurrentScope, tt as apiCallMetricSchema, un as cacheRecordingSchema, ur as runInExistingEvalScope, ut as llmCallMetricSchema, vn as traceAttributeDisplaySchema, vt as trialSelectionModeSchema, wn as traceSpanWarningSchema, wt as caseDetailSchema, xn as traceSpanErrorSchema, xt as getCaseRowCaseKey, yn as traceDisplayConfigSchema, yt as buildCaseKey, z as simulateLlmCallCost, zn as hashCacheKey, zt as manualInputFieldDescriptorSchema } from "./runOrchestration-C4o5TcIu.mjs";
+import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-CwGcJYWe.mjs";
+import "./src--13_4uDG.mjs";
 export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, advanceEvalTime, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, applyDerivedCallAttributes, assertionFailureSchema, buildCaseKey, buildEvalKey, buildTraceTree, cacheDebugKeyEntrySchema, cacheDebugKeyFileSchema, cacheEntrySchema, cacheEntryWithDebugKeySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, cleanupStagedManualInputFiles, columnDefSchema, columnFormatSchema, columnKindSchema, configReloadStateSchema, configReloadStatusSchema, createRunRequestSchema, createRunner, defaultConfigKeySchema, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, deserializeCacheRecording, deserializeCacheValue, discoveryIssueSchema, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalColumnOverrideSchema, evalColumnsSchema, evalDeriveConfigSchema, evalExpect, evalFreshnessStatusSchema, evalLog, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, fileRefSchema, getCaseRowCaseKey, getCaseRowEvalKey, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalStartTime, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, jsonCellSchema, llmCallCostCurrencySchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallPricingRateSchema, llmCallPricingSchema, llmCallsConfigSchema, manualInputBooleanFieldSchema, manualInputDescriptorSchema, manualInputFieldDescriptorSchema, manualInputFileValueSchema, manualInputJsonFieldSchema, manualInputMultilineFieldSchema, manualInputNumberFieldSchema, manualInputSelectFieldSchema, manualInputSelectOptionSchema, manualInputTextFieldSchema, materializeManualInputFiles, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, readManualInputFile, removeDefaultConfigSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runLogEntrySchema, runLogLevelSchema, runLogLocationSchema, runLogPhaseSchema, runLogsConfigSchema, runManifestSchema, runSummarySchema, scoreTraceSchema, serializeCacheRecording, serializeCacheValue, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, spanCacheOptionsSchema, sseEnvelopeSchema, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };