@ls-stack/agent-eval 0.36.0 → 0.37.0

package/dist/runChild.mjs

@@ -1,4 +1,4 @@
-import { A as createRunRequestSchema, C as loadConfig, D as createFsCacheStore, It as manualInputDescriptorSchema, K as runManifestSchema, Ot as evalStatsConfigSchema, Qt as evalChartsConfigSchema, Xn as configureEvalRunLogs, q as runSummarySchema, r as getTargetEvals$1, t as executeRun, vt as buildEvalKey, wn as columnDefSchema, x as parseEvalDiscovery } from "./runOrchestration-V1TxX8es.mjs";
+import { At as evalStatsConfigSchema, C as parseEvalDiscovery, En as columnDefSchema, J as runManifestSchema, M as createRunRequestSchema, Qn as configureEvalRunLogs, Rt as manualInputDescriptorSchema, T as loadConfig, Y as runSummarySchema, bt as buildEvalKey, en as evalChartsConfigSchema, k as createFsCacheStore, p as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-C4o5TcIu.mjs";
 import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";
@@ -14,6 +14,7 @@ const evalMetaSchema = z.object({
 	sourceFingerprint: z.string().nullable(),
 	columnDefs: z.array(columnDefSchema),
 	caseCount: z.number().nullable(),
+	caseIds: z.array(z.string()).optional(),
 	stats: evalStatsConfigSchema.optional(),
 	charts: evalChartsConfigSchema.optional(),
 	manualInputDescriptor: manualInputDescriptorSchema.optional(),
@@ -27,6 +28,7 @@ const runChildContextSchema = z.object({
 	summary: runSummarySchema,
 	evals: z.array(evalMetaSchema).optional()
 });
+let activeContext;
 function sendMessage(message) {
 	if (process.send === void 0) return;
 	process.send(message);
@@ -93,6 +95,7 @@ async function main() {
 		process.exit(1);
 	});
 	const context = await readContext(process.argv[2]);
+	activeContext = context;
 	process.chdir(context.workspaceRoot);
 	const config = await loadConfig();
 	configureEvalRunLogs({ captureConsole: config.runLogs?.captureConsole !== false });
@@ -153,7 +156,46 @@ async function main() {
 		evals: [...evals.values()]
 	});
 }
-await main();
+async function handleFatalRunChildError(error) {
+	const message = formatUnknownErrorDetails(error);
+	process.exitCode = 1;
+	console.error(message);
+	if (activeContext === void 0) return;
+	const endedAt = (/* @__PURE__ */ new Date()).toISOString();
+	await persistRunState({
+		runDir: activeContext.runDir,
+		manifest: {
+			...activeContext.manifest,
+			status: "error",
+			endedAt
+		},
+		summary: {
+			...activeContext.summary,
+			status: "error",
+			errorMessage: message
+		},
+		cases: [],
+		caseDetails: /* @__PURE__ */ new Map(),
+		listeners: /* @__PURE__ */ new Set()
+	});
+	sendMessage({
+		type: "event",
+		event: {
+			type: "run.error",
+			runId: activeContext.manifest.id,
+			timestamp: endedAt,
+			payload: { message }
+		}
+	});
+}
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
+await main().catch(async (error) => {
+	await handleFatalRunChildError(error);
+});
 process.disconnect();
 //#endregion
 export {};

package/dist/{runOrchestration-V1TxX8es.mjs → runOrchestration-C4o5TcIu.mjs}

@@ -2980,6 +2980,8 @@ const evalSummarySchema = z.object({
 	currentCommitSha: z.string().nullable(),
 	columnDefs: z.array(columnDefSchema),
 	caseCount: z.number().nullable(),
+	/** Authored case ids discovered for this eval, when case generation has run. */
+	caseIds: z.array(z.string()).optional(),
 	lastRunStatus: z.enum([
 		"pass",
 		"fail",
@@ -3626,6 +3628,11 @@ const runManifestSchema = z.object({
 		"cancelled",
 		"error"
 	]),
+	/**
+	* Temporary runs are persisted like normal runs, but are deleted before the
+	* next run starts. Older persisted runs default to durable history.
+	*/
+	temporary: z.boolean().optional().default(false),
 	startedAt: z.string(),
 	endedAt: z.string().nullable(),
 	/**
@@ -4521,6 +4528,11 @@ const createRunRequestSchema = z.object({
 	}),
 	trials: z.number().min(1),
 	/**
+	* Persist this run as temporary history. Temporary runs are visible while
+	* present, then deleted before the next run of any kind starts.
+	*/
+	temporary: z.boolean().optional(),
+	/**
 	* Optional cache controls for the run. When omitted, the cache is used in
 	* its default read-through / write-on-miss mode.
 	*/
@@ -6188,6 +6200,27 @@ function runTouchesEval(params) {
 	if (params.target.mode === "evalIds") return params.target.evalKeys?.includes(params.evalKey) ?? params.target.evalIds?.includes(params.evalId ?? params.evalKey) ?? false;
 	return false;
 }
+async function deleteTemporaryRuns(params) {
+	let deletedRuns = 0;
+	for (const [runId, run] of [...params.runs]) {
+		if (run.manifest.temporary !== true) continue;
+		if (run.manifest.status === "running") {
+			const endedAt = /* @__PURE__ */ new Date();
+			run.manifest.status = "cancelled";
+			run.manifest.endedAt = endedAt.toISOString();
+			run.summary.status = "cancelled";
+			run.summary.totalDurationMs = endedAt.getTime() - new Date(run.manifest.startedAt).getTime();
+			params.cancelRunningRun(run);
+		}
+		params.runs.delete(runId);
+		await rm(run.runDir, {
+			recursive: true,
+			force: true
+		});
+		deletedRuns += 1;
+	}
+	return deletedRuns;
+}
 async function recomputeEvalStatusesInRuns(params) {
 	let updatedRuns = 0;
 	for (const run of params.runs) {
@@ -6384,6 +6417,12 @@ function encodeCaseDetailFileName(caseId) {
 	return encodeURIComponent(caseId);
 }
 //#endregion
+//#region ../runner/src/stackFormatting.ts
+const orphanedAnsiSgrPattern = /\[(?:\d{1,3}(?:;\d{1,3})*)?m/g;
+function stripTerminalControlCodes(value) {
+	return stripVTControlCharacters(value).replaceAll(orphanedAnsiSgrPattern, "");
+}
+//#endregion
 //#region ../runner/src/moduleIsolation.ts
 const isolationParam = "agent-evals-isolate";
 const pathSegmentSeparatorPattern = /[\\/]+/;
@@ -6474,12 +6513,6 @@ async function runWithModuleIsolation(context, fn) {
 	return await isolationStorage.run(context, fn);
 }
 //#endregion
-//#region ../runner/src/stackFormatting.ts
-const orphanedAnsiSgrPattern = /\[(?:\d{1,3}(?:;\d{1,3})*)?m/g;
-function stripTerminalControlCodes(value) {
-	return stripVTControlCharacters(value).replaceAll(orphanedAnsiSgrPattern, "");
-}
-//#endregion
 //#region ../runner/src/runExecution.ts
 function filterEvalCases(cases, caseIds) {
 	if (!caseIds || caseIds.length === 0) return cases;
@@ -7061,6 +7094,8 @@ async function executeRun({ runState, request, runDir, config, cacheStore, lastR
 							const duplicateCaseIds = findDuplicateCaseIds(runnableCases);
 							if (duplicateCaseIds.length > 0) throw new Error(`Duplicate case id${duplicateCaseIds.length === 1 ? "" : "s"} in ${evalMeta.filePath}#${evalMeta.id}: ${duplicateCaseIds.join(", ")}`);
 							const cases = filterEvalCases(runnableCases, request.target.caseIds);
+							evalMeta.caseCount = runnableCases.length;
+							evalMeta.caseIds = runnableCases.map((evalCase) => evalCase.id);
 							runState.summary.totalCases += cases.length;
 							const defaultConfig = resolveEvalDefaultConfig({
 								evalDef,
@@ -7269,4 +7304,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { apiCallMetricSchema as $, getCurrentScope as $n, cacheDebugKeyEntrySchema as $t, createRunRequestSchema as A, repoFileRefSchema as An, runLogEntrySchema as At, getNestedAttribute as B, deserializeCacheValue as Bn, manualInputNumberFieldSchema as Bt, loadConfig as C, cellValueSchema as Cn, caseRowSchema as Ct, createFsCacheStore as D, fileRefSchema as Dn, evalStatItemSchema as Dt, validateCharts as E, columnKindSchema as En, evalStatAggregateSchema as Et, extractApiCalls as F, evalSpan as Fn, manualInputBooleanFieldSchema as Ft, deriveStatusFromChildStatuses as G, readManualInputFile as Gn, evalChartAxisSchema as Gt, getEvalDisplayStatus as H, serializeCacheValue as Hn, manualInputSelectOptionSchema as Ht, extractLlmCalls as I, evalTracer as In, manualInputDescriptorSchema as It, DEFAULT_API_CALLS_CONFIG as J, advanceEvalTime as Jn, evalChartConfigSchema as Jt, runManifestSchema as K, evalExpect as Kn, evalChartBuiltinMetricSchema as Kt, simulateLlmCallCost as L, hashCacheKey as Ln, manualInputFieldDescriptorSchema as Lt, sseEnvelopeSchema as M, z$1 as Mn, runLogLocationSchema as Mt, extractCacheEntries as N, buildTraceTree as Nn, runLogPhaseSchema as Nt, configReloadStateSchema as O, jsonCellSchema as On, evalStatsConfigSchema as Ot, extractCacheHits as P, captureEvalSpanError as Pn, scoreTraceSchema as Pt, apiCallMetricPlacementSchema as Q, evalLog as Qn, evalChartsConfigSchema as Qt, simulateTokenAllocation as R, hashCacheKeySync as Rn, manualInputJsonFieldSchema as Rt, resolveEvalDefaultConfig as S, traceSpanWarningSchema as Sn, caseDetailSchema as St, normalizeScoreDef as T, columnFormatSchema as Tn, evalFreshnessStatusSchema as Tt, deriveScopedSummaryFromCases as U, repoFile as Un, manualInputTextFieldSchema as Ut, getEvalTitle as V, serializeCacheRecording as Vn, manualInputSelectFieldSchema as Vt, deriveStatusFromCaseRows as W, manualInputFileValueSchema as Wn, evalChartAggregateSchema as Wt, agentEvalsConfigSchema as X, configureEvalRunLogs as Xn, evalChartTooltipExtraSchema as Xt, DEFAULT_LLM_CALLS_CONFIG as Y, appendToEvalOutput as Yn, evalChartMetricSchema as Yt, apiCallMetricFormatSchema as Z, evalAssert as Zn, evalChartTypeSchema as Zt, buildManualInputDescriptor as _, traceDisplayConfigSchema as _n, buildCaseKey as _t, getLastRunStatuses as a, cacheModeSchema as an, nextEvalId as ar, llmCallCostCurrencySchema as at, loadEvalModule as b, traceSpanKindSchema as bn, getCaseRowEvalKey as bt, loadPersistedRunSnapshots as c, cacheRecordingSchema as cn, runInExistingEvalScope as cr, llmCallMetricSchema as ct, persistRunState as d, spanCacheOptionsSchema as dn, startEvalBackgroundJob as dr, llmCallsConfigSchema as dt, cacheDebugKeyFileSchema as en, getEvalCaseInput as er, apiCallsConfigSchema as et, recomputeEvalStatusesInRuns as f, traceCacheRefSchema as fn, defineEval as fr, removeDefaultConfigSchema as ft, resolveArtifactPath as g, traceAttributeDisplaySchema as gn, trialSelectionModeSchema as gt, resolveTracePresentation as h, traceAttributeDisplayPlacementSchema as hn, runLogsConfigSchema as ht, generateRunId as i, cacheListItemSchema as in, mergeEvalOutput as ir, evalDeriveConfigSchema as it, updateManualScoreRequestSchema as j, runArtifactRefSchema as jn, runLogLevelSchema as jt, configReloadStatusSchema as k, numberDisplayOptionsSchema as kn, evalSummarySchema as kt, nextShortIdFromSnapshots as l, cacheStatusSchema as ln, setEvalOutput as lr, llmCallPricingRateSchema as lt, runTouchesEval as m, traceAttributeDisplayInputSchema as mn, resolveLlmCallsConfig as mt, getTargetEvalKeys as n, cacheEntryWithDebugKeySchema as nn, incrementEvalOutput as nr, evalColumnOverrideSchema as nt, getLatestRunInfos as o, cacheOperationTypeSchema as on, runInEvalRuntimeScope as or, llmCallMetricFormatSchema as ot, recomputePersistedCaseStatus as p, traceAttributeDisplayFormatSchema as pn, getEvalRegistry as pr, resolveApiCallsConfig as pt, runSummarySchema as q, EvalAssertionError as qn, evalChartColorSchema as qt, getTargetEvals as r, cacheFileSchema as rn, isInEvalScope as rr, evalColumnsSchema as rt, loadPersistedRunSnapshot as s, cacheRecordingOpSchema as sn, runInEvalScope as sr, llmCallMetricPlacementSchema as st, executeRun as t, cacheEntrySchema as tn, getEvalStartTime as tr, defaultConfigKeySchema as tt, persistCaseDetail as u, serializedCacheSpanSchema as un, setScopeCacheContext as ur, llmCallPricingSchema as ut, parseManualInputValues as v, traceDisplayInputConfigSchema as vn, buildEvalKey as vt, buildDeclaredColumnDefs as w, columnDefSchema as wn, discoveryIssueSchema as wt, parseEvalDiscovery as x, traceSpanSchema as xn, assertionFailureSchema as xt, deriveEvalFreshness as y, traceSpanErrorSchema as yn, getCaseRowCaseKey as yt, applyDerivedCallAttributes as z, deserializeCacheRecording as zn, manualInputMultilineFieldSchema as zt };
+export { apiCallMetricFormatSchema as $, evalAssert as $n, evalChartTypeSchema as $t, configReloadStateSchema as A, jsonCellSchema as An, evalStatsConfigSchema as At, simulateTokenAllocation as B, hashCacheKeySync as Bn, manualInputJsonFieldSchema as Bt, parseEvalDiscovery as C, traceSpanSchema as Cn, assertionFailureSchema as Ct, normalizeScoreDef as D, columnFormatSchema as Dn, evalFreshnessStatusSchema as Dt, buildDeclaredColumnDefs as E, columnDefSchema as En, discoveryIssueSchema as Et, extractCacheEntries as F, buildTraceTree as Fn, runLogPhaseSchema as Ft, deriveScopedSummaryFromCases as G, repoFile as Gn, manualInputTextFieldSchema as Gt, getNestedAttribute as H, deserializeCacheValue as Hn, manualInputNumberFieldSchema as Ht, extractCacheHits as I, captureEvalSpanError as In, scoreTraceSchema as It, runManifestSchema as J, evalExpect as Jn, evalChartBuiltinMetricSchema as Jt, deriveStatusFromCaseRows as K, manualInputFileValueSchema as Kn, evalChartAggregateSchema as Kt, extractApiCalls as L, evalSpan as Ln, manualInputBooleanFieldSchema as Lt, createRunRequestSchema as M, repoFileRefSchema as Mn, runLogEntrySchema as Mt, updateManualScoreRequestSchema as N, runArtifactRefSchema as Nn, runLogLevelSchema as Nt, validateCharts as O, columnKindSchema as On, evalStatAggregateSchema as Ot, sseEnvelopeSchema as P, z$1 as Pn, runLogLocationSchema as Pt, agentEvalsConfigSchema as Q, configureEvalRunLogs as Qn, evalChartTooltipExtraSchema as Qt, extractLlmCalls as R, evalTracer as Rn, manualInputDescriptorSchema as Rt, loadEvalModule as S, traceSpanKindSchema as Sn, getCaseRowEvalKey as St, loadConfig as T, cellValueSchema as Tn, caseRowSchema as Tt, getEvalTitle as U, serializeCacheRecording as Un, manualInputSelectFieldSchema as Ut, applyDerivedCallAttributes as V, deserializeCacheRecording as Vn, manualInputMultilineFieldSchema as Vt, getEvalDisplayStatus as W, serializeCacheValue as Wn, manualInputSelectOptionSchema as Wt, DEFAULT_API_CALLS_CONFIG as X, advanceEvalTime as Xn, evalChartConfigSchema as Xt, runSummarySchema as Y, EvalAssertionError as Yn, evalChartColorSchema as Yt, DEFAULT_LLM_CALLS_CONFIG as Z, appendToEvalOutput as Zn, evalChartMetricSchema as Zt, resolveTracePresentation as _, traceAttributeDisplayPlacementSchema as _n, runLogsConfigSchema as _t, generateRunId as a, cacheFileSchema as an, isInEvalScope as ar, evalColumnsSchema as at, parseManualInputValues as b, traceDisplayInputConfigSchema as bn, buildEvalKey as bt, loadPersistedRunSnapshot as c, cacheOperationTypeSchema as cn, runInEvalRuntimeScope as cr, llmCallMetricFormatSchema as ct, persistCaseDetail as d, cacheStatusSchema as dn, setEvalOutput as dr, llmCallPricingRateSchema as dt, evalChartsConfigSchema as en, evalLog as er, apiCallMetricPlacementSchema as et, deleteTemporaryRuns as f, serializedCacheSpanSchema as fn, setScopeCacheContext as fr, llmCallPricingSchema as ft, runTouchesEval as g, traceAttributeDisplayInputSchema as gn, resolveLlmCallsConfig as gt, recomputePersistedCaseStatus as h, traceAttributeDisplayFormatSchema as hn, getEvalRegistry as hr, resolveApiCallsConfig as ht, stripTerminalControlCodes as i, cacheEntryWithDebugKeySchema as in, incrementEvalOutput as ir, evalColumnOverrideSchema as it, configReloadStatusSchema as j, numberDisplayOptionsSchema as jn, evalSummarySchema as jt, createFsCacheStore as k, fileRefSchema as kn, evalStatItemSchema as kt, loadPersistedRunSnapshots as l, cacheRecordingOpSchema as ln, runInEvalScope as lr, llmCallMetricPlacementSchema as lt, recomputeEvalStatusesInRuns as m, traceCacheRefSchema as mn, defineEval as mr, removeDefaultConfigSchema as mt, getTargetEvalKeys as n, cacheDebugKeyFileSchema as nn, getEvalCaseInput as nr, apiCallsConfigSchema as nt, getLastRunStatuses as o, cacheListItemSchema as on, mergeEvalOutput as or, evalDeriveConfigSchema as ot, persistRunState as p, spanCacheOptionsSchema as pn, startEvalBackgroundJob as pr, llmCallsConfigSchema as pt, deriveStatusFromChildStatuses as q, readManualInputFile as qn, evalChartAxisSchema as qt, getTargetEvals as r, cacheEntrySchema as rn, getEvalStartTime as rr, defaultConfigKeySchema as rt, getLatestRunInfos as s, cacheModeSchema as sn, nextEvalId as sr, llmCallCostCurrencySchema as st, executeRun as t, cacheDebugKeyEntrySchema as tn, getCurrentScope as tr, apiCallMetricSchema as tt, nextShortIdFromSnapshots as u, cacheRecordingSchema as un, runInExistingEvalScope as ur, llmCallMetricSchema as ut, resolveArtifactPath as v, traceAttributeDisplaySchema as vn, trialSelectionModeSchema as vt, resolveEvalDefaultConfig as w, traceSpanWarningSchema as wn, caseDetailSchema as wt, deriveEvalFreshness as x, traceSpanErrorSchema as xn, getCaseRowCaseKey as xt, buildManualInputDescriptor as y, traceDisplayConfigSchema as yn, buildCaseKey as yt, simulateLlmCallCost as z, hashCacheKey as zn, manualInputFieldDescriptorSchema as zt };

package/dist/{runner-BCs5rzej.mjs → runner-BTH8m_Er.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-Dg3abrOv.mjs";
-import "./src-DBypR4TV.mjs";
+import { n as createRunner } from "./cli-CwGcJYWe.mjs";
+import "./src--13_4uDG.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{runner-znY6PY1M.mjs → runner-LqeHPID6.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-BCs5rzej.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-BTH8m_Er.mjs";
 export { getRunnerInstance, initRunner };

package/dist/src--13_4uDG.mjs

@@ -0,0 +1,3 @@
+import "./runOrchestration-C4o5TcIu.mjs";
+import "./cli-CwGcJYWe.mjs";
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.36.0",
+  "version": "0.37.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -32,8 +32,8 @@
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/shared": "0.0.1",
-    "@agent-evals/sdk": "0.0.1"
+    "@agent-evals/sdk": "0.0.1",
+    "@agent-evals/shared": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",

package/skills/agent-eval/SKILL.md

@@ -27,7 +27,13 @@ display rules), read the TypeScript declarations shipped with the package:
 - Unfiltered `agent-evals run` is disabled by default; use `--eval` or `--case`
   for targeted CLI runs. Set `allowCliRunAll: true` in
   `agent-evals.config.ts` to opt into run-all CLI behavior. The web UI can
-  still run grouped evals and confirms before starting more than five.
+  still run grouped evals and confirms before starting more than five. On a
+  single eval page, the Run chevron can open a picker to run specific authored
+  case ids; those case-picked runs are temporary by default and can be made
+  durable in the modal.
+- `agent-evals run --temporary` persists a run like normal history, but deletes
+  it before the next run starts. Temporary runs appear in `show-runs` and the UI
+  while present; normal runs are never deleted by temporary-run cleanup.
 - `agent-evals app` watches `agent-evals.config.ts` and reloads config in
   place when the runner is idle. If config changes during an active run, the UI
   shows a pending reload banner and blocks new runs until the current run
@@ -507,6 +513,8 @@ Run output lives under `.agent-evals/runs/<run-id>/` and cache entries under
 `.agent-evals/cache/<eval-id>.json`. Files in a run directory include run
 metadata, a run summary, per-case results, and per-case trace JSON. Inspect
 these when debugging persisted output, costs, columns, traces, or failures.
+Temporary runs use the same directory layout, but are removed before the next
+run of any kind starts.
 
 Use `agent-evals show-runs` when you need stable file
 paths before reading saved output: