@ls-stack/agent-eval 0.14.0 → 0.15.0

package/dist/{app-DKWm1oxc.mjs → app-B7FUWsVm.mjs}

@@ -1,6 +1,6 @@
-import { C as updateManualScoreRequestSchema, S as createRunRequestSchema } from "./runOrchestration-H0pSUl3I.mjs";
-import "./src-BgGL7DDp.mjs";
-import { t as getRunnerInstance } from "./runner-DLnj18MO.mjs";
+import { C as createRunRequestSchema, w as updateManualScoreRequestSchema } from "./runOrchestration-B3fYtpKo.mjs";
+import "./src-jaOlXwb5.mjs";
+import { t as getRunnerInstance } from "./runner-vunKoSBu.mjs";
 import { readFile } from "node:fs/promises";
 import { dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod/v4";

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-CMiCEQ-3.mjs";
+import { t as runCli } from "./cli-B-sCTyz8.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/{cli-CMiCEQ-3.mjs → cli-B-sCTyz8.mjs}

@@ -1,4 +1,4 @@
-import { A as getEvalDisplayStatus, F as runSummarySchema, J as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, fn as getEvalRegistry, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, j as deriveScopedSummaryFromCases, k as getEvalTitle, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, q as resolveApiCallsConfig, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-H0pSUl3I.mjs";
+import { A as getEvalTitle, I as runSummarySchema, J as resolveApiCallsConfig, M as deriveScopedSummaryFromCases, S as createFsCacheStore, Y as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as buildDeclaredColumnDefs, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, g as setLatestRunInfoMap, h as getTargetEvalIds, hn as getEvalRegistry, i as getLatestRunInfos, j as getEvalDisplayStatus, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as parseEvalMetas, x as normalizeScoreDef, y as loadConfig } from "./runOrchestration-B3fYtpKo.mjs";
 import { createHash } from "node:crypto";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join, relative, resolve } from "node:path";
@@ -82,98 +82,6 @@ function validateCharts(params) {
 	};
 }
 //#endregion
-//#region ../runner/src/discovery.ts
-const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
-const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
-function parseEvalMetas(filePath, content) {
-	const metas = [];
-	let searchIndex = 0;
-	while (searchIndex < content.length) {
-		const defineEvalIndex = content.indexOf("defineEval", searchIndex);
-		if (defineEvalIndex === -1) break;
-		const extracted = extractDefineEvalObject(content, defineEvalIndex);
-		if (!extracted) {
-			searchIndex = defineEvalIndex + 10;
-			continue;
-		}
-		const id = evalIdMatchRegex.exec(extracted.objectText)?.[1];
-		if (id !== void 0) {
-			const result = {
-				filePath,
-				id
-			};
-			const title = evalTitleMatchRegex.exec(extracted.objectText)?.[1];
-			if (title !== void 0) result.title = title;
-			metas.push(result);
-		}
-		searchIndex = extracted.nextIndex;
-	}
-	return metas;
-}
-function extractDefineEvalObject(content, defineEvalIndex) {
-	const openParenIndex = content.indexOf("(", defineEvalIndex);
-	if (openParenIndex === -1) return void 0;
-	const objectStartIndex = content.indexOf("{", openParenIndex);
-	if (objectStartIndex === -1) return void 0;
-	let depth = 0;
-	let quote;
-	let inBlockComment = false;
-	let inLineComment = false;
-	let isEscaped = false;
-	for (let index = objectStartIndex; index < content.length; index++) {
-		const currentChar = content[index];
-		const nextChar = content[index + 1];
-		if (inLineComment) {
-			if (currentChar === "\n") inLineComment = false;
-			continue;
-		}
-		if (inBlockComment) {
-			if (currentChar === "*" && nextChar === "/") {
-				inBlockComment = false;
-				index++;
-			}
-			continue;
-		}
-		if (quote) {
-			if (isEscaped) {
-				isEscaped = false;
-				continue;
-			}
-			if (currentChar === "\\") {
-				isEscaped = true;
-				continue;
-			}
-			if (currentChar === quote) quote = void 0;
-			continue;
-		}
-		if (currentChar === "/" && nextChar === "/") {
-			inLineComment = true;
-			index++;
-			continue;
-		}
-		if (currentChar === "/" && nextChar === "*") {
-			inBlockComment = true;
-			index++;
-			continue;
-		}
-		if (currentChar === "\"" || currentChar === "'" || currentChar === "`") {
-			quote = currentChar;
-			continue;
-		}
-		if (currentChar === "{") {
-			depth++;
-			continue;
-		}
-		if (currentChar === "}") {
-			depth--;
-			if (depth === 0) return {
-				nextIndex: index + 1,
-				objectText: content.slice(objectStartIndex, index + 1)
-			};
-		}
-	}
-}
-//#endregion
 //#region ../runner/src/gitState.ts
 function runGitCommand(workspaceRoot, args) {
 	const result = spawnSync("git", args, {
@@ -316,7 +224,10 @@ function upsertFinishedCase(runState, caseDetail, caseRow) {
 function applyChildEvalMetas(evals, childMetas) {
 	for (const childMeta of childMetas) {
 		const evalMeta = evals.get(childMeta.id);
-		if (evalMeta === void 0) continue;
+		if (evalMeta === void 0) {
+			evals.set(childMeta.id, childMeta);
+			continue;
+		}
 		evalMeta.columnDefs = childMeta.columnDefs;
 		evalMeta.caseCount = childMeta.caseCount;
 		evalMeta.stats = childMeta.stats;
@@ -728,8 +639,7 @@ function createRunner({ watchForChanges = true } = {}) {
 				workspaceRoot,
 				runDir,
 				manifest,
-				summary,
-				evals: getSortedEvalMetas()
+				summary
 			};
 			await writeFile(join(runDir, "run-child-context.json"), JSON.stringify(childContext, null, 2));
 			startRunChild({
@@ -1059,8 +969,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-DKWm1oxc.mjs");
-	const runnerModule = await import("./runner-Dx1sMCbh.mjs");
+	const appModule = await import("./app-B7FUWsVm.mjs");
+	const runnerModule = await import("./runner-Dt-Ynv6s.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();

package/dist/index.d.mts

@@ -2720,6 +2720,14 @@ type EvalCaseScope = {
   caseCacheRefs: TraceCacheRef[]; /** Background promises that should settle before the case scope finalizes. */
   pendingBackgroundJobs: Set<Promise<unknown>>;
 };
+/**
+ * Runtime phase currently owned by the eval runner.
+ *
+ * `null` means the current async execution is outside an eval run. `env`
+ * covers run-time module/environment loading, including top-level code in
+ * modules imported while a run is being prepared.
+ */
+type EvalRuntimeScope = 'env' | 'cases' | 'eval' | 'derive' | 'outputsSchema' | 'scorer';
 /** Error thrown when an eval assertion fails during case execution. */
 declare class EvalAssertionError extends Error {
   constructor(message: string);
@@ -2727,12 +2735,14 @@ declare class EvalAssertionError extends Error {
 /** Return the current eval scope for the active async context, if any. */
 declare function getCurrentScope(): EvalCaseScope | undefined;
 /**
- * Return whether the current async execution is inside an active eval case.
+ * Return the current eval runner phase for this async execution.
  *
- * This is useful for shared workflow code that wants to branch on eval-only
- * behavior without importing or inspecting the full eval scope.
+ * Returns `null` outside eval-owned work, `env` while the runner is loading
+ * eval modules for a run, `cases` while generating cases, `eval` while running
+ * case `execute`, `derive` while deriving outputs from traces, `outputsSchema`
+ * while validating outputs, and `scorer` while computing scores.
  */
-declare function isInEvalScope(): boolean;
+declare function isInEvalScope(): EvalRuntimeScope | null;
 /**
  * Register background work that should settle before eval finalization.
  *
@@ -2762,8 +2772,18 @@ type RunInEvalScopeOptions = {
   /** Authored input for the active eval case. */input?: unknown; /** Stable prefix used when generating scoped IDs with `nextEvalId()`. */
   idPrefix?: string; /** Cache adapter + mode attached to the scope before `fn` runs. */
   cacheContext?: CacheScopeContext; /** Whether registered background jobs should settle before scope finalizes. */
-  waitForBackgroundJobs?: boolean;
+  waitForBackgroundJobs?: boolean; /** Eval runner phase exposed through `isInEvalScope()`. Defaults to `eval`. */
+  runtimeScope?: EvalRuntimeScope;
 };
+/** Execute a callback while `isInEvalScope()` reports a runner phase. */
+declare function runInEvalRuntimeScope<T>(runtimeScope: EvalRuntimeScope, fn: () => Promise<T> | T): Promise<T>;
+/**
+ * Execute a callback with an existing case scope and a specific runner phase.
+ *
+ * Runner-internal helper for post-execute phases that still need access to the
+ * completed case scope through output, trace, assertion, and input helpers.
+ */
+declare function runInExistingEvalScope<T>(scope: EvalCaseScope, runtimeScope: EvalRuntimeScope, fn: () => Promise<T> | T): Promise<T>;
 /**
  * Execute a callback inside a fresh eval case scope and capture its outputs,
  * trace data, and terminal error state.
@@ -3135,4 +3155,4 @@ declare function createRunner({
  */
 declare function runCli(argv: string[]): Promise<void>;
 //#endregion
-export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalRuntimeScope, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as evalFreshnessStatusSchema, $t as evalAssert, A as getEvalDisplayStatus, At as traceDisplayInputConfigSchema, B as apiCallMetricPlacementSchema, Bt as jsonCellSchema, C as updateManualScoreRequestSchema, Ct as spanCacheOptionsSchema, D as extractLlmCalls, Dt as traceAttributeDisplayPlacementSchema, E as extractApiCalls, Et as traceAttributeDisplayInputSchema, F as runSummarySchema, Ft as cellValueSchema, G as llmCallMetricSchema, Gt as buildTraceTree, H as apiCallsConfigSchema, Ht as repoFileRefSchema, I as DEFAULT_API_CALLS_CONFIG, It as columnDefSchema, J as resolveLlmCallsConfig, Jt as evalTracer, K as llmCallsConfigSchema, Kt as captureEvalSpanError, L as DEFAULT_LLM_CALLS_CONFIG, Lt as columnFormatSchema, M as deriveStatusFromCaseRows, Mt as traceSpanKindSchema, N as deriveStatusFromChildStatuses, Nt as traceSpanSchema, O as getNestedAttribute, Ot as traceAttributeDisplaySchema, P as runManifestSchema, Pt as traceSpanWarningSchema, Q as caseRowSchema, Qt as appendToEvalOutput, R as agentEvalsConfigSchema, Rt as columnKindSchema, S as createRunRequestSchema, St as serializedCacheSpanSchema, T as extractCacheHits, Tt as traceAttributeDisplayFormatSchema, U as llmCallMetricFormatSchema, Ut as runArtifactRefSchema, V as apiCallMetricSchema, Vt as numberDisplayOptionsSchema, W as llmCallMetricPlacementSchema, Wt as z, X as assertionFailureSchema, Xt as hashCacheKeySync, Y as trialSelectionModeSchema, Yt as hashCacheKey, Z as caseDetailSchema, Zt as EvalAssertionError, _t as cacheModeSchema, an as nextEvalId, at as evalChartAggregateSchema, bt as cacheRecordingSchema, cn as setScopeCacheContext, ct as evalChartColorSchema, dn as defineEval, dt as evalChartTooltipExtraSchema, en as getCurrentScope, et as evalStatAggregateSchema, fn as getEvalRegistry, ft as evalChartTypeSchema, gt as cacheListItemSchema, ht as cacheFileSchema, in as mergeEvalOutput, it as scoreTraceSchema, j as deriveScopedSummaryFromCases, jt as traceSpanErrorSchema, k as getEvalTitle, kt as traceDisplayConfigSchema, ln as startEvalBackgroundJob, lt as evalChartConfigSchema, mt as cacheEntrySchema, nn as incrementEvalOutput, nt as evalStatsConfigSchema, on as runInEvalScope, ot as evalChartAxisSchema, pt as evalChartsConfigSchema, q as resolveApiCallsConfig, qt as evalSpan, rn as isInEvalScope, rt as evalSummarySchema, sn as setEvalOutput, st as evalChartBuiltinMetricSchema, tn as getEvalCaseInput, tt as evalStatItemSchema, un as repoFile, ut as evalChartMetricSchema, vt as cacheOperationTypeSchema, w as sseEnvelopeSchema, wt as traceCacheRefSchema, xt as cacheStatusSchema, yt as cacheRecordingOpSchema, z as apiCallMetricFormatSchema, zt as fileRefSchema } from "./runOrchestration-H0pSUl3I.mjs";
-import { n as createRunner, t as runCli } from "./cli-CMiCEQ-3.mjs";
-import "./src-BgGL7DDp.mjs";
-export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+import { $ as caseRowSchema, $t as appendToEvalOutput, A as getEvalTitle, At as traceDisplayConfigSchema, B as apiCallMetricFormatSchema, Bt as fileRefSchema, C as createRunRequestSchema, Ct as serializedCacheSpanSchema, D as extractApiCalls, Dt as traceAttributeDisplayInputSchema, E as extractCacheHits, Et as traceAttributeDisplayFormatSchema, F as runManifestSchema, Ft as traceSpanWarningSchema, G as llmCallMetricPlacementSchema, Gt as z, H as apiCallMetricSchema, Ht as numberDisplayOptionsSchema, I as runSummarySchema, It as cellValueSchema, J as resolveApiCallsConfig, Jt as evalSpan, K as llmCallMetricSchema, Kt as buildTraceTree, L as DEFAULT_API_CALLS_CONFIG, Lt as columnDefSchema, M as deriveScopedSummaryFromCases, Mt as traceSpanErrorSchema, N as deriveStatusFromCaseRows, Nt as traceSpanKindSchema, O as extractLlmCalls, Ot as traceAttributeDisplayPlacementSchema, P as deriveStatusFromChildStatuses, Pt as traceSpanSchema, Q as caseDetailSchema, Qt as EvalAssertionError, R as DEFAULT_LLM_CALLS_CONFIG, Rt as columnFormatSchema, St as cacheStatusSchema, T as sseEnvelopeSchema, Tt as traceCacheRefSchema, U as apiCallsConfigSchema, Ut as repoFileRefSchema, V as apiCallMetricPlacementSchema, Vt as jsonCellSchema, W as llmCallMetricFormatSchema, Wt as runArtifactRefSchema, X as trialSelectionModeSchema, Xt as hashCacheKey, Y as resolveLlmCallsConfig, Yt as evalTracer, Z as assertionFailureSchema, Zt as hashCacheKeySync, _t as cacheListItemSchema, an as mergeEvalOutput, at as scoreTraceSchema, bt as cacheRecordingOpSchema, cn as runInEvalScope, ct as evalChartBuiltinMetricSchema, dn as setScopeCacheContext, dt as evalChartMetricSchema, en as evalAssert, et as evalFreshnessStatusSchema, fn as startEvalBackgroundJob, ft as evalChartTooltipExtraSchema, gt as cacheFileSchema, hn as getEvalRegistry, ht as cacheEntrySchema, in as isInEvalScope, it as evalSummarySchema, j as getEvalDisplayStatus, jt as traceDisplayInputConfigSchema, k as getNestedAttribute, kt as traceAttributeDisplaySchema, ln as runInExistingEvalScope, lt as evalChartColorSchema, mn as defineEval, mt as evalChartsConfigSchema, nn as getEvalCaseInput, nt as evalStatItemSchema, on as nextEvalId, ot as evalChartAggregateSchema, pn as repoFile, pt as evalChartTypeSchema, q as llmCallsConfigSchema, qt as captureEvalSpanError, rn as incrementEvalOutput, rt as evalStatsConfigSchema, sn as runInEvalRuntimeScope, st as evalChartAxisSchema, tn as getCurrentScope, tt as evalStatAggregateSchema, un as setEvalOutput, ut as evalChartConfigSchema, vt as cacheModeSchema, w as updateManualScoreRequestSchema, wt as spanCacheOptionsSchema, xt as cacheRecordingSchema, yt as cacheOperationTypeSchema, z as agentEvalsConfigSchema, zt as columnKindSchema } from "./runOrchestration-B3fYtpKo.mjs";
+import { n as createRunner, t as runCli } from "./cli-B-sCTyz8.mjs";
+import "./src-jaOlXwb5.mjs";
+export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.mjs

@@ -1,7 +1,9 @@
-import { F as runSummarySchema, It as columnDefSchema, P as runManifestSchema, S as createRunRequestSchema, nt as evalStatsConfigSchema, pt as evalChartsConfigSchema, t as executeRun, v as loadConfig, x as createFsCacheStore } from "./runOrchestration-H0pSUl3I.mjs";
+import { C as createRunRequestSchema, F as runManifestSchema, I as runSummarySchema, Lt as columnDefSchema, S as createFsCacheStore, mt as evalChartsConfigSchema, rt as evalStatsConfigSchema, t as executeRun, v as parseEvalMetas, y as loadConfig } from "./runOrchestration-B3fYtpKo.mjs";
 import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
+import { relative } from "node:path";
 import { z } from "zod/v4";
+import { glob } from "glob";
 //#region ../runner/src/runChild.ts
 const evalMetaSchema = z.object({
 	id: z.string(),
@@ -20,7 +22,7 @@ const runChildContextSchema = z.object({
 	runDir: z.string(),
 	manifest: runManifestSchema,
 	summary: runSummarySchema,
-	evals: z.array(evalMetaSchema)
+	evals: z.array(evalMetaSchema).optional()
 });
 function sendMessage(message) {
 	if (process.send === void 0) return;
@@ -37,6 +39,38 @@ function getTargetEvals(params) {
 	if (params.request.target.evalIds && params.request.target.evalIds.length > 0) return params.request.target.evalIds.map((id) => params.evals.get(id)).filter((entry) => entry !== void 0);
 	return [...params.evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
 }
+function toWorkspaceRelativePath(params) {
+	return relative(params.workspaceRoot, params.filePath).replaceAll("\\", "/");
+}
+async function discoverRunEvals(params) {
+	const discovered = [];
+	for (const pattern of params.config.include) {
+		const files = await glob(pattern, {
+			cwd: params.workspaceRoot,
+			absolute: true
+		});
+		discovered.push(...files);
+	}
+	const evals = /* @__PURE__ */ new Map();
+	for (const filePath of discovered) {
+		const source = await readFile(filePath, "utf-8");
+		const sourceFingerprint = getSourceFingerprint(source);
+		const metas = parseEvalMetas(filePath, source);
+		for (const meta of metas) evals.set(meta.id, {
+			id: meta.id,
+			title: meta.title,
+			filePath: toWorkspaceRelativePath({
+				filePath: meta.filePath,
+				workspaceRoot: params.workspaceRoot
+			}),
+			sourceFilePath: meta.filePath,
+			sourceFingerprint,
+			columnDefs: [],
+			caseCount: null
+		});
+	}
+	return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+}
 async function readContext(contextPath) {
 	if (contextPath === void 0) throw new Error("Missing run child context path");
 	return runChildContextSchema.parse(JSON.parse(await readFile(contextPath, "utf-8")));
@@ -54,7 +88,11 @@ async function main() {
 		maxEntriesPerNamespace: config.cache?.maxEntriesPerNamespace ?? config.cache?.maxEntriesPerEval,
 		maxEntriesByNamespace: config.cache?.maxEntriesByNamespace
 	});
-	const evals = new Map(context.evals.map((evalMeta) => [evalMeta.id, evalMeta]));
+	const evalMetas = await discoverRunEvals({
+		config,
+		workspaceRoot: context.workspaceRoot
+	});
+	const evals = new Map(evalMetas.map((evalMeta) => [evalMeta.id, evalMeta]));
 	const lastRunStatusMap = /* @__PURE__ */ new Map();
 	const latestRunInfoMap = /* @__PURE__ */ new Map();
 	await executeRun({

package/dist/{runOrchestration-H0pSUl3I.mjs → runOrchestration-B3fYtpKo.mjs}

@@ -46,7 +46,9 @@ function repoFile(path, mimeType) {
 //#endregion
 //#region ../sdk/src/runtime.ts
 const scopeStorage = new AsyncLocalStorage();
+const runtimeScopeStorage = new AsyncLocalStorage();
 let activeEvalScopeCount = 0;
+let activeEvalRuntimeScopeCount = 0;
 /** Error thrown when an eval assertion fails during case execution. */
 var EvalAssertionError = class extends Error {
 	constructor(message) {
@@ -60,13 +62,16 @@ function getCurrentScope() {
 	return scopeStorage.getStore();
 }
 /**
-* Return whether the current async execution is inside an active eval case.
+* Return the current eval runner phase for this async execution.
 *
-* This is useful for shared workflow code that wants to branch on eval-only
-* behavior without importing or inspecting the full eval scope.
+* Returns `null` outside eval-owned work, `env` while the runner is loading
+* eval modules for a run, `cases` while generating cases, `eval` while running
+* case `execute`, `derive` while deriving outputs from traces, `outputsSchema`
+* while validating outputs, and `scorer` while computing scores.
 */
 function isInEvalScope() {
-	return getCurrentScope() !== void 0;
+	if (activeEvalRuntimeScopeCount === 0) return null;
+	return runtimeScopeStorage.getStore() ?? null;
 }
 function registerBackgroundJobInScope(scope, promise) {
 	const trackedPromise = promise.then(() => {
@@ -122,6 +127,31 @@ function getEvalCaseInput(path = void 0) {
 function setScopeCacheContext(scope, context) {
 	scope.cacheContext = context;
 }
+/** Execute a callback while `isInEvalScope()` reports a runner phase. */
+async function runInEvalRuntimeScope(runtimeScope, fn) {
+	activeEvalRuntimeScopeCount++;
+	try {
+		return await runtimeScopeStorage.run(runtimeScope, fn);
+	} finally {
+		activeEvalRuntimeScopeCount--;
+	}
+}
+/**
+* Execute a callback with an existing case scope and a specific runner phase.
+*
+* Runner-internal helper for post-execute phases that still need access to the
+* completed case scope through output, trace, assertion, and input helpers.
+*/
+async function runInExistingEvalScope(scope, runtimeScope, fn) {
+	activeEvalScopeCount++;
+	try {
+		return await scopeStorage.run(scope, async () => {
+			return await runInEvalRuntimeScope(runtimeScope, fn);
+		});
+	} finally {
+		activeEvalScopeCount--;
+	}
+}
 /**
 * Execute a callback inside a fresh eval case scope and capture its outputs,
 * trace data, and terminal error state.
@@ -144,29 +174,24 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		caseCacheRefs: [],
 		pendingBackgroundJobs: /* @__PURE__ */ new Set()
 	};
-	activeEvalScopeCount++;
-	try {
-		return await scopeStorage.run(scope, async () => {
-			try {
-				const result = await fn();
-				if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
-				return {
-					result,
-					scope,
-					error: void 0
-				};
-			} catch (error) {
-				if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
-				return {
-					result: void 0,
-					scope,
-					error: error instanceof Error ? error : new Error(String(error))
-				};
-			}
-		});
-	} finally {
-		activeEvalScopeCount--;
-	}
+	return await runInExistingEvalScope(scope, options.runtimeScope ?? "eval", async () => {
+		try {
+			const result = await fn();
+			if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
+			return {
+				result,
+				scope,
+				error: void 0
+			};
+		} catch (error) {
+			if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
+			return {
+				result: void 0,
+				scope,
+				error: error instanceof Error ? error : new Error(String(error))
+			};
+		}
+	});
 }
 /**
 * Return the next deterministic ID for the active eval case execution.
@@ -4886,6 +4911,98 @@ async function loadConfig() {
 	}
 }
 //#endregion
+//#region ../runner/src/discovery.ts
+const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
+const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
+function parseEvalMetas(filePath, content) {
+	const metas = [];
+	let searchIndex = 0;
+	while (searchIndex < content.length) {
+		const defineEvalIndex = content.indexOf("defineEval", searchIndex);
+		if (defineEvalIndex === -1) break;
+		const extracted = extractDefineEvalObject(content, defineEvalIndex);
+		if (!extracted) {
+			searchIndex = defineEvalIndex + 10;
+			continue;
+		}
+		const id = evalIdMatchRegex.exec(extracted.objectText)?.[1];
+		if (id !== void 0) {
+			const result = {
+				filePath,
+				id
+			};
+			const title = evalTitleMatchRegex.exec(extracted.objectText)?.[1];
+			if (title !== void 0) result.title = title;
+			metas.push(result);
+		}
+		searchIndex = extracted.nextIndex;
+	}
+	return metas;
+}
+function extractDefineEvalObject(content, defineEvalIndex) {
+	const openParenIndex = content.indexOf("(", defineEvalIndex);
+	if (openParenIndex === -1) return void 0;
+	const objectStartIndex = content.indexOf("{", openParenIndex);
+	if (objectStartIndex === -1) return void 0;
+	let depth = 0;
+	let quote;
+	let inBlockComment = false;
+	let inLineComment = false;
+	let isEscaped = false;
+	for (let index = objectStartIndex; index < content.length; index++) {
+		const currentChar = content[index];
+		const nextChar = content[index + 1];
+		if (inLineComment) {
+			if (currentChar === "\n") inLineComment = false;
+			continue;
+		}
+		if (inBlockComment) {
+			if (currentChar === "*" && nextChar === "/") {
+				inBlockComment = false;
+				index++;
+			}
+			continue;
+		}
+		if (quote) {
+			if (isEscaped) {
+				isEscaped = false;
+				continue;
+			}
+			if (currentChar === "\\") {
+				isEscaped = true;
+				continue;
+			}
+			if (currentChar === quote) quote = void 0;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "/") {
+			inLineComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "/" && nextChar === "*") {
+			inBlockComment = true;
+			index++;
+			continue;
+		}
+		if (currentChar === "\"" || currentChar === "'" || currentChar === "`") {
+			quote = currentChar;
+			continue;
+		}
+		if (currentChar === "{") {
+			depth++;
+			continue;
+		}
+		if (currentChar === "}") {
+			depth--;
+			if (depth === 0) return {
+				nextIndex: index + 1,
+				objectText: content.slice(objectStartIndex, index + 1)
+			};
+		}
+	}
+}
+//#endregion
 //#region ../runner/src/evalModuleLoader.ts
 /**
 * Import one eval module with a cache key derived from its current source so
@@ -5280,6 +5397,7 @@ const isolationParam = "agent-evals-isolate";
 const pathSegmentSeparatorPattern = /[\\/]+/;
 const isolationStorage = new AsyncLocalStorage();
 const activeIsolationRoots = /* @__PURE__ */ new Map();
+const clearedRequireCacheKeys = /* @__PURE__ */ new Set();
 let hooksRegistered = false;
 const requireFromRunner = createRequire(import.meta.url);
 const agentPackageUrlBySpecifier = new Map([
@@ -5305,7 +5423,10 @@ function getIsolationKeyFromParent(parentURL) {
 }
 function isWorkspaceFile(url, workspaceRoot) {
 	if (url.protocol !== "file:") return false;
-	const relativePath = relative(workspaceRoot, fileURLToPath(url));
+	return isWorkspaceFilePath(fileURLToPath(url), workspaceRoot);
+}
+function isWorkspaceFilePath(filePath, workspaceRoot) {
+	const relativePath = relative(workspaceRoot, filePath);
 	if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) return false;
 	const segments = relativePath.split(pathSegmentSeparatorPattern);
 	return !segments.includes("node_modules") && !segments.includes(".agent-evals");
@@ -5340,15 +5461,23 @@ function registerModuleIsolationHooks() {
 		};
 	} });
 }
+function clearWorkspaceRequireCacheOnce(context) {
+	if (clearedRequireCacheKeys.has(context.key)) return;
+	clearedRequireCacheKeys.add(context.key);
+	for (const filePath of Object.keys(requireFromRunner.cache)) if (isWorkspaceFilePath(filePath, context.workspaceRoot)) delete requireFromRunner.cache[filePath];
+}
 /**
 * Execute module loading and eval code with fresh workspace module URLs.
 *
 * Node does not expose an ESM cache reset API, so the runner appends a
-* run-scoped query parameter to workspace file imports. Package imports are
-* left alone so SDK singletons, such as the eval registry, remain shared.
+* run-scoped query parameter to workspace file imports. CommonJS modules use
+* `require.cache` behind ESM imports, so workspace entries are cleared once per
+* run. Package imports are left alone so SDK singletons, such as the eval
+* registry, remain shared.
 */
 async function runWithModuleIsolation(context, fn) {
 	registerModuleIsolationHooks();
+	clearWorkspaceRequireCacheOnce(context);
 	activeIsolationRoots.set(context.key, context.workspaceRoot);
 	return await isolationStorage.run(context, fn);
 }
@@ -5476,20 +5605,26 @@ async function runCase(params) {
 	const traceTree = buildTraceTree(scope.spans, scope.checkpoints);
 	const nonAssertError = executeError && !(executeError instanceof EvalAssertionError) ? executeError : null;
 	if (executeError instanceof EvalAssertionError && scope.assertionFailures.length === 0) scope.assertionFailures.push(toAssertionFailure(executeError.message, executeError));
-	if (!nonAssertError && evalDef.deriveFromTracing) try {
-		const derived = await callWithUnknownResult(evalDef.deriveFromTracing, [{
-			trace: traceTree,
-			input: evalCase.input,
-			case: evalCase
-		}]);
-		if (!isRecord(derived)) throw new Error("deriveFromTracing must return an object");
-		for (const [key, value] of Object.entries(derived)) if (!(key in scope.outputs)) scope.outputs[key] = value;
-	} catch (e) {
-		const message = `deriveFromTracing threw: ${e instanceof Error ? e.message : String(e)}`;
-		scope.assertionFailures.push(toAssertionFailure(message, e instanceof Error ? e : void 0));
+	if (!nonAssertError && evalDef.deriveFromTracing) {
+		const { deriveFromTracing } = evalDef;
+		try {
+			const derived = await runInExistingEvalScope(scope, "derive", async () => {
+				return await callWithUnknownResult(deriveFromTracing, [{
+					trace: traceTree,
+					input: evalCase.input,
+					case: evalCase
+				}]);
+			});
+			if (!isRecord(derived)) throw new Error("deriveFromTracing must return an object");
+			for (const [key, value] of Object.entries(derived)) if (!(key in scope.outputs)) scope.outputs[key] = value;
+		} catch (e) {
+			const message = `deriveFromTracing threw: ${e instanceof Error ? e.message : String(e)}`;
+			scope.assertionFailures.push(toAssertionFailure(message, e instanceof Error ? e : void 0));
+		}
 	}
 	if (!nonAssertError && evalDef.outputsSchema) {
-		const parsedOutputs = evalDef.outputsSchema.safeParse(getOutputsSchemaInput(evalDef.outputsSchema, scope.outputs));
+		const { outputsSchema } = evalDef;
+		const parsedOutputs = await runInExistingEvalScope(scope, "outputsSchema", () => outputsSchema.safeParse(getOutputsSchemaInput(outputsSchema, scope.outputs)));
 		if (parsedOutputs.success) scope.outputs = {
 			...scope.outputs,
 			...parsedOutputs.data
@@ -5511,6 +5646,7 @@ async function runCase(params) {
 		}, {
 			input: evalCase.input,
 			idPrefix: `${scopedIdPrefix}-score-${toStableIdSegment(key)}`,
+			runtimeScope: "scorer",
 			cacheContext: cacheAdapter ? {
 				adapter: cacheAdapter,
 				mode: cacheMode,
@@ -5791,12 +5927,19 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			} catch {
 				codeFingerprint = "";
 			}
-			if (codeFingerprint.length > 0) runState.manifest.evalSourceFingerprints[evalMeta.id] = codeFingerprint;
-			else delete runState.manifest.evalSourceFingerprints[evalMeta.id];
+			if (codeFingerprint.length > 0) {
+				runState.manifest.evalSourceFingerprints[evalMeta.id] = codeFingerprint;
+				evalMeta.sourceFingerprint = codeFingerprint;
+			} else {
+				delete runState.manifest.evalSourceFingerprints[evalMeta.id];
+				evalMeta.sourceFingerprint = null;
+			}
 			try {
 				const registry = getEvalRegistry();
 				await runWithModuleIsolation(moduleIsolation, async () => {
-					await loadEvalModule(evalFilePath, codeFingerprint);
+					await runInEvalRuntimeScope("env", async () => {
+						await loadEvalModule(evalFilePath, codeFingerprint);
+					});
 				});
 				const entry = registry.get(evalMeta.id);
 				if (!entry) {
@@ -5807,87 +5950,89 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 					continue;
 				}
 				await runWithModuleIsolation(moduleIsolation, async () => {
-					await entry.use(async (evalDef) => {
-						const cases = filterEvalCases(resolveRunnableEvalCases({
-							cases: typeof evalDef.cases === "function" ? await evalDef.cases() : evalDef.cases ?? [],
-							evalId: evalMeta.id
-						}), request.target.evalIds, request.target.caseIds, evalMeta.id);
-						runState.summary.totalCases += cases.length;
-						const accumulatedColumns = /* @__PURE__ */ new Map();
-						const evalCaseRows = [];
-						const preparedCases = [];
-						const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
-						const manualScoreKeys = Object.freeze(Object.keys(evalDef.manualScores ?? {}));
-						const preparedEval = {
-							evalMeta,
-							accumulatedColumns,
-							evalCaseRows,
-							preparedCases,
-							scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
-							mergeColumns: (columns) => {
-								mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
-							}
-						};
-						preparedEvals.push(preparedEval);
-						for (const evalCase of cases) {
-							const trialResults = [];
-							const preparedCase = {
-								caseId: evalCase.id,
-								trialResults,
-								finalized: false
+					await runInEvalRuntimeScope("cases", async () => {
+						await entry.use(async (evalDef) => {
+							const cases = filterEvalCases(resolveRunnableEvalCases({
+								cases: typeof evalDef.cases === "function" ? await evalDef.cases() : evalDef.cases ?? [],
+								evalId: evalMeta.id
+							}), request.target.evalIds, request.target.caseIds, evalMeta.id);
+							runState.summary.totalCases += cases.length;
+							const accumulatedColumns = /* @__PURE__ */ new Map();
+							const evalCaseRows = [];
+							const preparedCases = [];
+							const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
+							const manualScoreKeys = Object.freeze(Object.keys(evalDef.manualScores ?? {}));
+							const preparedEval = {
+								evalMeta,
+								accumulatedColumns,
+								evalCaseRows,
+								preparedCases,
+								scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
+								mergeColumns: (columns) => {
+									mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
+								}
 							};
-							preparedCases.push(preparedCase);
-							for (let trial = 0; trial < request.trials; trial++) {
-								const bufferedCacheStore = cacheEnabled && cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
-								queuedCases.push({
-									execute: async ({ startTime, globalTraceDisplay }) => {
-										const { caseDetail, caseRowUpdate } = await runCase({
-											evalDef,
-											evalId: evalMeta.id,
-											evalCase,
-											globalTraceDisplay,
-											trial,
-											startTime,
-											cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
-											cacheMode,
-											codeFingerprint,
-											moduleIsolation,
-											evalFilePath,
-											workspaceRoot,
-											artifactDir: join(runDir, "artifacts"),
-											runId: runState.manifest.id
-										});
-										return {
-											caseDetail,
-											caseRow: {
-												caseId: evalCase.id,
+							preparedEvals.push(preparedEval);
+							for (const evalCase of cases) {
+								const trialResults = [];
+								const preparedCase = {
+									caseId: evalCase.id,
+									trialResults,
+									finalized: false
+								};
+								preparedCases.push(preparedCase);
+								for (let trial = 0; trial < request.trials; trial++) {
+									const bufferedCacheStore = cacheEnabled && cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
+									queuedCases.push({
+										execute: async ({ startTime, globalTraceDisplay }) => {
+											const { caseDetail, caseRowUpdate } = await runCase({
+												evalDef,
 												evalId: evalMeta.id,
-												status: caseRowUpdate.status ?? "pending",
-												latencyMs: caseRowUpdate.latencyMs ?? null,
-												columns: caseRowUpdate.columns ?? {},
-												trial
-											}
-										};
-									},
-									onComplete: async ({ caseDetail, caseRow }) => {
-										trialResults.push({
-											caseDetail,
-											caseRow,
-											bufferedCacheStore
-										});
-										if (trialResults.length !== request.trials) return;
-										await finalizePreparedCase({
-											runState,
-											runDir,
-											preparedEval,
-											preparedCase,
-											onCaseFinished,
-											emitEvent
-										});
-									}
-								});
+												evalCase,
+												globalTraceDisplay,
+												trial,
+												startTime,
+												cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
+												cacheMode,
+												codeFingerprint,
+												moduleIsolation,
+												evalFilePath,
+												workspaceRoot,
+												artifactDir: join(runDir, "artifacts"),
+												runId: runState.manifest.id
+											});
+											return {
+												caseDetail,
+												caseRow: {
+													caseId: evalCase.id,
+													evalId: evalMeta.id,
+													status: caseRowUpdate.status ?? "pending",
+													latencyMs: caseRowUpdate.latencyMs ?? null,
+													columns: caseRowUpdate.columns ?? {},
+													trial
+												}
+											};
+										},
+										onComplete: async ({ caseDetail, caseRow }) => {
+											trialResults.push({
+												caseDetail,
+												caseRow,
+												bufferedCacheStore
+											});
+											if (trialResults.length !== request.trials) return;
+											await finalizePreparedCase({
+												runState,
+												runDir,
+												preparedEval,
+												preparedCase,
+												onCaseFinished,
+												emitEvent
+											});
+										}
+									});
+								}
 							}
-						}
+						});
 					});
 				});
 			} catch (error) {
@@ -5995,4 +6140,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { evalFreshnessStatusSchema as $, evalAssert as $t, getEvalDisplayStatus as A, traceDisplayInputConfigSchema as At, apiCallMetricPlacementSchema as B, jsonCellSchema as Bt, updateManualScoreRequestSchema as C, spanCacheOptionsSchema as Ct, extractLlmCalls as D, traceAttributeDisplayPlacementSchema as Dt, extractApiCalls as E, traceAttributeDisplayInputSchema as Et, runSummarySchema as F, cellValueSchema as Ft, llmCallMetricSchema as G, buildTraceTree as Gt, apiCallsConfigSchema as H, repoFileRefSchema as Ht, DEFAULT_API_CALLS_CONFIG as I, columnDefSchema as It, resolveLlmCallsConfig as J, evalTracer as Jt, llmCallsConfigSchema as K, captureEvalSpanError as Kt, DEFAULT_LLM_CALLS_CONFIG as L, columnFormatSchema as Lt, deriveStatusFromCaseRows as M, traceSpanKindSchema as Mt, deriveStatusFromChildStatuses as N, traceSpanSchema as Nt, getNestedAttribute as O, traceAttributeDisplaySchema as Ot, runManifestSchema as P, traceSpanWarningSchema as Pt, caseRowSchema as Q, appendToEvalOutput as Qt, agentEvalsConfigSchema as R, columnKindSchema as Rt, createRunRequestSchema as S, serializedCacheSpanSchema as St, extractCacheHits as T, traceAttributeDisplayFormatSchema as Tt, llmCallMetricFormatSchema as U, runArtifactRefSchema as Ut, apiCallMetricSchema as V, numberDisplayOptionsSchema as Vt, llmCallMetricPlacementSchema as W, z$1 as Wt, assertionFailureSchema as X, hashCacheKeySync as Xt, trialSelectionModeSchema as Y, hashCacheKey as Yt, caseDetailSchema as Z, EvalAssertionError as Zt, loadEvalModule as _, cacheModeSchema as _t, loadPersistedRunSnapshot as a, nextEvalId as an, evalChartAggregateSchema as at, normalizeScoreDef as b, cacheRecordingSchema as bt, persistCaseDetail as c, setScopeCacheContext as cn, evalChartColorSchema as ct, recomputePersistedCaseStatus as d, defineEval as dn, evalChartTooltipExtraSchema as dt, getCurrentScope as en, evalStatAggregateSchema as et, runTouchesEval as f, getEvalRegistry as fn, evalChartTypeSchema as ft, setLatestRunInfoMap as g, cacheListItemSchema as gt, getTargetEvalIds as h, cacheFileSchema as ht, getLatestRunInfos as i, mergeEvalOutput as in, scoreTraceSchema as it, deriveScopedSummaryFromCases as j, traceSpanErrorSchema as jt, getEvalTitle as k, traceDisplayConfigSchema as kt, persistRunState as l, startEvalBackgroundJob as ln, evalChartConfigSchema as lt, buildEvalSummary as m, cacheEntrySchema as mt, generateRunId as n, incrementEvalOutput as nn, evalStatsConfigSchema as nt, loadPersistedRunSnapshots as o, runInEvalScope as on, evalChartAxisSchema as ot, resolveArtifactPath as p, evalChartsConfigSchema as pt, resolveApiCallsConfig as q, evalSpan as qt, getLastRunStatuses as r, isInEvalScope as rn, evalSummarySchema as rt, nextShortIdFromSnapshots as s, setEvalOutput as sn, evalChartBuiltinMetricSchema as st, executeRun as t, getEvalCaseInput as tn, evalStatItemSchema as tt, recomputeEvalStatusesInRuns as u, repoFile as un, evalChartMetricSchema as ut, loadConfig as v, cacheOperationTypeSchema as vt, sseEnvelopeSchema as w, traceCacheRefSchema as wt, createFsCacheStore as x, cacheStatusSchema as xt, buildDeclaredColumnDefs as y, cacheRecordingOpSchema as yt, apiCallMetricFormatSchema as z, fileRefSchema as zt };
+export { caseRowSchema as $, appendToEvalOutput as $t, getEvalTitle as A, traceDisplayConfigSchema as At, apiCallMetricFormatSchema as B, fileRefSchema as Bt, createRunRequestSchema as C, serializedCacheSpanSchema as Ct, extractApiCalls as D, traceAttributeDisplayInputSchema as Dt, extractCacheHits as E, traceAttributeDisplayFormatSchema as Et, runManifestSchema as F, traceSpanWarningSchema as Ft, llmCallMetricPlacementSchema as G, z$1 as Gt, apiCallMetricSchema as H, numberDisplayOptionsSchema as Ht, runSummarySchema as I, cellValueSchema as It, resolveApiCallsConfig as J, evalSpan as Jt, llmCallMetricSchema as K, buildTraceTree as Kt, DEFAULT_API_CALLS_CONFIG as L, columnDefSchema as Lt, deriveScopedSummaryFromCases as M, traceSpanErrorSchema as Mt, deriveStatusFromCaseRows as N, traceSpanKindSchema as Nt, extractLlmCalls as O, traceAttributeDisplayPlacementSchema as Ot, deriveStatusFromChildStatuses as P, traceSpanSchema as Pt, caseDetailSchema as Q, EvalAssertionError as Qt, DEFAULT_LLM_CALLS_CONFIG as R, columnFormatSchema as Rt, createFsCacheStore as S, cacheStatusSchema as St, sseEnvelopeSchema as T, traceCacheRefSchema as Tt, apiCallsConfigSchema as U, repoFileRefSchema as Ut, apiCallMetricPlacementSchema as V, jsonCellSchema as Vt, llmCallMetricFormatSchema as W, runArtifactRefSchema as Wt, trialSelectionModeSchema as X, hashCacheKey as Xt, resolveLlmCallsConfig as Y, evalTracer as Yt, assertionFailureSchema as Z, hashCacheKeySync as Zt, loadEvalModule as _, cacheListItemSchema as _t, loadPersistedRunSnapshot as a, mergeEvalOutput as an, scoreTraceSchema as at, buildDeclaredColumnDefs as b, cacheRecordingOpSchema as bt, persistCaseDetail as c, runInEvalScope as cn, evalChartBuiltinMetricSchema as ct, recomputePersistedCaseStatus as d, setScopeCacheContext as dn, evalChartMetricSchema as dt, evalAssert as en, evalFreshnessStatusSchema as et, runTouchesEval as f, startEvalBackgroundJob as fn, evalChartTooltipExtraSchema as ft, setLatestRunInfoMap as g, cacheFileSchema as gt, getTargetEvalIds as h, getEvalRegistry as hn, cacheEntrySchema as ht, getLatestRunInfos as i, isInEvalScope as in, evalSummarySchema as it, getEvalDisplayStatus as j, traceDisplayInputConfigSchema as jt, getNestedAttribute as k, traceAttributeDisplaySchema as kt, persistRunState as l, runInExistingEvalScope as ln, evalChartColorSchema as lt, buildEvalSummary as m, defineEval as mn, evalChartsConfigSchema as mt, generateRunId as n, getEvalCaseInput as nn, evalStatItemSchema as nt, loadPersistedRunSnapshots as o, nextEvalId as on, evalChartAggregateSchema as ot, resolveArtifactPath as p, repoFile as pn, evalChartTypeSchema as pt, llmCallsConfigSchema as q, captureEvalSpanError as qt, getLastRunStatuses as r, incrementEvalOutput as rn, evalStatsConfigSchema as rt, nextShortIdFromSnapshots as s, runInEvalRuntimeScope as sn, evalChartAxisSchema as st, executeRun as t, getCurrentScope as tn, evalStatAggregateSchema as tt, recomputeEvalStatusesInRuns as u, setEvalOutput as un, evalChartConfigSchema as ut, parseEvalMetas as v, cacheModeSchema as vt, updateManualScoreRequestSchema as w, spanCacheOptionsSchema as wt, normalizeScoreDef as x, cacheRecordingSchema as xt, loadConfig as y, cacheOperationTypeSchema as yt, agentEvalsConfigSchema as z, columnKindSchema as zt };

package/dist/{runner-Dx1sMCbh.mjs → runner-Dt-Ynv6s.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-DLnj18MO.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-vunKoSBu.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-DLnj18MO.mjs → runner-vunKoSBu.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-CMiCEQ-3.mjs";
-import "./src-BgGL7DDp.mjs";
+import { n as createRunner } from "./cli-B-sCTyz8.mjs";
+import "./src-jaOlXwb5.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/src-jaOlXwb5.mjs

@@ -0,0 +1,3 @@
+import "./runOrchestration-B3fYtpKo.mjs";
+import "./cli-B-sCTyz8.mjs";
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -13,7 +13,8 @@
     }
   },
   "files": [
-    "dist"
+    "dist",
+    "skills"
   ],
   "tsdown": {
     "clean": true,

package/skills/agent-eval/SKILL.md

@@ -0,0 +1,400 @@
+---
+name: agent-eval
+description: Create, run, and maintain TypeScript evals with @ls-stack/agent-eval. Use when adding eval coverage for an LLM or agent workflow, updating *.eval.ts files, checking eval results, configuring agent-evals.config.ts, inspecting saved .agent-evals run artifacts, or wiring product source code with evalTracer spans.
+---
+
+# Agent Eval
+
+Local-first, UI-first eval runner for LLM and agent systems. Evals are strict
+TypeScript modules named `*.eval.ts`, discovered from `agent-evals.config.ts`,
+and executed through the CLI (`agent-evals run`) or the web UI
+(`agent-evals app`). Runs persist to `.agent-evals/` so results, traces, and
+caches survive across processes.
+
+This skill covers the mental model and conventions. For exhaustive field lists
+(config options, eval shape, column formats, score/chart/stats options, trace
+display rules), read the TypeScript declarations shipped with the package:
+
+- `AgentEvalsConfig`, `EvalDefinition`, `EvalCase`, `EvalOutputs`,
+  `EvalColumnOverride`, `EvalScoreDef`, `EvalManualScoreDef`,
+  `EvalTraceTree`, `TraceSpanInfo`, and `z` are exported from
+  `@ls-stack/agent-eval`.
+- `.d.ts` files land in `node_modules/@ls-stack/agent-eval/dist/`.
+- CLI surface: `agent-evals --help` and `agent-evals <command> --help`.
+  Unknown help targets exit non-zero instead of falling back to global help.
+- The CLI automatically loads `.env` from the current workspace. Shell-provided
+  environment variables win; pass `--no-env` to disable `.env` loading once.
+
+Assume that enumerated tables in this document may lag behind the types —
+treat the types as source of truth when they disagree.
+
+## Where tracing lives
+
+**Tracing belongs in the product source code, not in the eval file.** The eval
+file wires up cases and scoring; the real `evalTracer.span(...)` calls sit
+inside the workflow, agent, or tool functions that both production and evals
+invoke.
+
+`evalTracer`, `evalSpan`, output helpers, and `evalAssert` are ambient no-ops
+when called outside an eval case scope, so leaving them in production paths is
+safe — they only record anything when the product code runs inside an eval's
+`execute`. Use `isInEvalScope()` to branch on eval-only behavior in shared code
+(e.g. skip a real network side effect): it returns `null` outside eval-owned
+work and returns `'env'`, `'cases'`, `'eval'`, `'derive'`, `'outputsSchema'`, or
+`'scorer'` during runner phases. Top-level modules imported while a run is being
+prepared see `'env'`; code called from `execute` sees `'eval'`. Use
+`getEvalCaseInput()` to read the current case input, or
+`getEvalCaseInput('customer.tier')` for nested dot-path access; outside a case
+scope it returns `undefined`. Use `nextEvalId()` inside eval-scoped code when a
+stable generated id is needed; it includes the eval file, eval id, case id, and
+a per-case sequence number, and throws outside an eval case scope.
+
+### Product code (instrumented once, reused everywhere)
+
+```ts
+// src/workflows/refundWorkflow.ts
+import {
+  appendToEvalOutput,
+  captureEvalSpanError,
+  evalAssert,
+  evalSpan,
+  evalTracer,
+  getEvalCaseInput,
+  incrementEvalOutput,
+  mergeEvalOutput,
+  nextEvalId,
+  setEvalOutput,
+  startEvalBackgroundJob,
+} from '@ls-stack/agent-eval';
+
+export async function runRefundWorkflow(input: RefundInput) {
+  return evalTracer.span(
+    { kind: 'agent', name: 'refund-workflow' },
+    async () => {
+      evalSpan.setAttribute('input', input);
+
+      const plan = await evalTracer.span(
+        {
+          kind: 'llm',
+          name: 'plan-refund',
+          cache: { key: { prompt: input.message, model: 'gpt-4o-mini' } },
+        },
+        async () => {
+          let text: string;
+          let usage: { inputTokens: number; outputTokens: number };
+          let costUsd: number;
+          try {
+            ({ text, usage, costUsd } = await llm.complete(input.message));
+          } catch (error) {
+            captureEvalSpanError(error);
+            ({ text, usage, costUsd } = await llm.completeWithFallback(
+              input.message,
+            ));
+          }
+          evalSpan.setAttributes({ model: 'gpt-4o-mini', usage });
+          const expectedLocale = getEvalCaseInput('locale');
+          if (typeof expectedLocale === 'string') {
+            evalSpan.setAttribute('expectedLocale', expectedLocale);
+          }
+          evalSpan.incrementAttribute('llmCalls', 1);
+          evalSpan.appendToAttribute('models', 'gpt-4o-mini');
+          incrementEvalOutput('costUsd', costUsd);
+          appendToEvalOutput('modelCalls', { model: 'gpt-4o-mini', costUsd });
+          return text;
+        },
+      );
+
+      const result = await applyRefund(plan);
+      const reviewId = nextEvalId();
+      setEvalOutput('response', result.finalText);
+      setEvalOutput('reviewId', reviewId);
+      mergeEvalOutput('metadata', { approved: result.approved });
+      evalAssert(result.approved, 'refund workflow should approve the case');
+      evalSpan.setAttribute('output', { result, reviewId });
+      return result;
+    },
+  );
+}
+```
+
+Span `kind` values are open-ended strings and are color-coded automatically in
+the UI for every kind used during the app session. Use familiar kinds such as
+`agent`, `tool`, `llm`, `api`, `retrieval`, `scorer`, or `checkpoint` when they
+fit, and preserve external tracer kinds such as `mastra.workflow.step` when they
+are more specific. The UI automatically promotes only the `input` and `output` span
+attributes. Use `traceDisplay` for other span attributes such as `model`,
+`usage`, or `costUsd`.
+
+Use `captureEvalSpanError(error)` for recoverable errors on the active
+`evalTracer.span(...)`, such as optional model/tool failures that fall back and
+continue. You can pass one error, multiple error arguments, or an array. The
+span is still marked `error`, and the UI renders captured errors in a dedicated
+span detail block with timing relative to the span. Pass `'warning'` or
+`{ level: 'warning' }` as the final argument for diagnostics that should be
+visible in span detail without changing an otherwise successful span's status.
+
+If a span callback throws, the SDK automatically marks that span as `error`,
+stores the thrown error on it, and rethrows so the case errors. Use that for
+terminal failures; use `captureEvalSpanError(...)` for recoverable failures that
+continue through fallback logic.
+
+Fire-and-forget spans started during `execute` are awaited before outputs,
+`deriveFromTracing`, scores, and trace data are finalized, so `void
+evalTracer.span(...)` is safe when the span result is not needed. Register
+non-span promises with `startEvalBackgroundJob(promise)`. The runner only waits
+for settlement; promise and span errors keep their normal behavior. Use
+`waitForBackgroundJob: false` on a span, or `waitForBackgroundJobs: false` on an
+eval definition, when background work should not delay finalization.
+
+For libraries or observability exporters that already emit span lifecycle
+events, use `evalTracer.startSpan(...)`, `evalTracer.updateSpan(...)`,
+`evalTracer.endSpan(...)`, or `evalTracer.recordSpan(...)` to translate those
+events into the eval trace tree without wrapping the upstream work in a
+callback. Pass the upstream span id and parent id when available so the UI keeps
+the original hierarchy.
+
+### Eval file (thin)
+
+```ts
+// evals/refund-workflow.eval.ts
+import { defineEval, z } from '@ls-stack/agent-eval';
+import { runRefundWorkflow } from '../src/workflows/refundWorkflow.ts';
+
+const outputsSchema = z.object({
+  response: z.string(),
+  costUsd: z.number().optional(),
+  toolCalls: z.number(),
+  llmTurns: z.number(),
+});
+type RefundOutputs = z.infer<typeof outputsSchema>;
+
+defineEval<RefundInput, RefundOutputs>({
+  id: 'refund-workflow',
+  cases: [
+    { id: 'simple-text', input: { message: 'I want a refund for order #123' } },
+  ],
+  outputsSchema,
+  execute: async ({ input }) => {
+    await runRefundWorkflow(input);
+  },
+  deriveFromTracing: ({ trace }) => ({
+    toolCalls: trace.findSpansByKind('tool').length,
+    llmTurns: trace.findSpansByKind('llm').length,
+  }),
+  scores: {
+    mentionsRefund: {
+      passThreshold: 1,
+      compute: ({ outputs }) => (/refund/i.test(outputs.response) ? 1 : 0),
+    },
+  },
+});
+```
+
+`execute` usually just calls the product code. Push any placeholder
+`evalTracer.span(...)` wrappers out of the eval and into the product module
+they describe so production runs get the same trajectory. Only keep tracing
+inside `execute` when the behavior being measured is eval-specific (e.g. a
+judge-only sub-step with no production analogue).
+
+Case `id` values anchor historical runs, caches, and manual scores — keep them
+stable. See `EvalDefinition` / `EvalCase` in the types for every supported
+field.
+
+## Scoring
+
+Every score returns a normalized `0..1` value. Pass/fail is per-score: a case
+fails if any score with `passThreshold` falls below it, if an assertion fails,
+or if the case errors. Scores without `passThreshold` are informational.
+
+Score functions run in their own trace scope, separate from the execution
+trace, so LLM-as-judge scorers can use `evalTracer.span(...)` and cached spans
+without polluting the agent trajectory. The case detail UI shows execution
+spans on **Trace** and scorer spans on **Scoring**. Outputs set inside a scorer
+stay private to that score.
+
+`manualScores` declares score columns that reviewers fill in the web UI after
+a run. Pending values keep the eval in an `unscored` state instead of failing.
+
+See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape
+(format, threshold, column overrides).
+
+## Outputs, columns, trace display
+
+- `setEvalOutput(key, value)` writes reviewable data for the case. Values are
+  plain data (strings, numbers, booleans, JSON-safe objects) plus native
+  `Blob`/`File` or `FileRef` variants for media columns. Inside `execute`,
+  prefer the context `setOutput(key, value)` helper when writing schema-backed
+  outputs; it is typed from the eval's outputs generic. Keep `setEvalOutput`
+  for shared workflow code that does not receive the execute context.
+- Use `incrementEvalOutput(key, delta)` for numeric totals,
+  `appendToEvalOutput(key, value)` for arrays that preserve existing scalar
+  values, and `mergeEvalOutput(key, patch)` for shallow object updates.
+  `evalSpan` has matching `incrementAttribute`, `appendToAttribute`, and
+  `mergeAttribute` helpers for span attributes.
+- `outputsSchema` validates final outputs after `execute` and
+  `deriveFromTracing`, before computed scores. For Zod object schemas, only
+  declared keys are passed to the schema; parsed fields merge back into the raw
+  output map, so defaults/transforms apply to configured fields and
+  unconfigured outputs stay visible as before. Validation failures fail the case
+  and skip computed scores. When you pass a narrowed outputs type as the second
+  `defineEval` generic, `outputsSchema` is required.
+- `columns` overrides the display for output and score keys (label, format,
+  alignment, visibility). The set of supported formats is declared by the
+  `ColumnFormat` union and `EvalColumnOverride` in the types.
+- `traceDisplay` promotes selected span attributes into the trace tree and
+  detail pane; it supports aggregation across subtrees (`scope`, `mode`) and
+  user-defined `transform(...)` for derived views (e.g. currency conversion).
+  See the `TraceDisplayInputConfig` type.
+- `llmCalls` (in `agent-evals.config.ts`) configures the LLM calls tab in the
+  case-run drawer. Defaults to `kind: 'llm'` spans with `model`, `usage.*`,
+  `costUsd`, `input`, `output`, etc. read from conventional attribute paths.
+  Override `kinds` to broaden the filter, override `attributes.<field>` for
+  non-default span shapes, and add entries to `metrics` to surface arbitrary
+  user metrics (`format: 'string' | 'number' | 'duration' | 'json' |
+'boolean'`, `placements: ['header' | 'body']`). The tab auto-hides when no
+  matching spans exist.
+- `apiCalls` (in `agent-evals.config.ts`) configures the API calls tab in the
+  case-run drawer. Defaults to `kind: 'api'`, `'http'`, `'http.client'`, and
+  `'fetch'` spans with `method`, `url`, `statusCode`, `request`, `response`,
+  `requestBody`, `responseBody`, `headers`, `durationMs`, and `error` read
+  from conventional attribute paths. Override `kinds` or
+  `attributes.<field>` for external tracers, and add `metrics` with the same
+  formats and placements as LLM-call metrics. The tab auto-hides when no
+  matching spans exist.
+
+Stats rows and history charts on the eval card are opt-in via `stats` /
+`charts` on the eval definition. Their shapes live in the types; no need to
+memorize the option set.
+
+## Cached operations
+
+Wrap a costly pure span in `cache: { key }` so later runs replay its recorded
+effects without re-executing:
+
+```ts
+await evalTracer.span(
+  {
+    kind: 'llm',
+    name: 'plan-refund',
+    cache: { key: { prompt: input.message, model: 'gpt-4o-mini' } },
+  },
+  async () => {
+    const result = await llm.complete(input.message);
+    evalSpan.setAttributes({ model: 'gpt-4o-mini', output: result });
+    incrementEvalOutput('costUsd', computeCost(result));
+    appendToEvalOutput('llmCalls', { model: 'gpt-4o-mini' });
+    return result;
+  },
+);
+```
+
+Use `evalTracer.cache(...)` for pure values that should not create their own
+trace span:
+
+```ts
+const context = await evalTracer.cache(
+  { name: 'receipt-audit-context', key: { orderId: input.orderId } },
+  async () => {
+    const result = await loadReceiptContext(input);
+    evalSpan.setAttribute('receiptContext', result);
+    evalSpan.mergeAttribute('receiptSummary', { orderId: input.orderId });
+    return result;
+  },
+);
+```
+
+Mental model:
+
+- Only SDK-mediated effects replay on a hit: sub-spans, checkpoints,
+  output helper calls, span attributes. External side
+  effects (HTTP, DB writes, file I/O) **do not** replay — cache only pure
+  functions of the key.
+- `evalTracer.cache(...)` does not create a span. When it runs inside an active
+  span, that span gets a `cache.refs` entry with the value cache name, key,
+  namespace, and hit/miss status. When called directly from the case body
+  (no surrounding span), the ref is recorded on the case detail's `cacheRefs`
+  array so spanless caches still appear in the UI's **Cache hits** tab, where
+  each hit can be expanded for inspection or deleted by namespace/key.
+- The cache key folds in a source-file fingerprint, so editing the eval busts
+  the cache automatically.
+- `cache.namespace` on spans or `namespace` on value caches can share entries
+  across operations/evals, but the source-file fingerprint still participates
+  in the final key. Shared namespaces are reusable across evals in the same
+  file; evals in different files miss even with the same namespace and key.
+- Cache keys should be deterministic primitives, arrays, and plain objects.
+  `Buffer`, `ArrayBuffer`, and typed arrays hash by bytes. Native `Blob`/`File`
+  keys use stable metadata by default (`type`, `size`, plus
+  `name`/`lastModified` for `File`) and do not read file bytes. Add
+  `serializeFileBytes: true` to a cached span or `evalTracer.cache(...)` call
+  when byte-level cache invalidation is required.
+- Cache entries are stored in inspectable owner files under
+  `.agent-evals/cache/<owner>.json`; each namespace is capped at 100 entries by
+  default. Configure `cache.maxEntriesPerNamespace` for the default cap and
+  `cache.maxEntriesByNamespace` for exact namespace-specific caps.
+- Cached payloads use advance serialization/deserialization with the Web API plugin set, so return values and
+  recorded SDK effects preserve richer built-ins such as `Date`, `Map`, `Set`,
+  typed arrays, `URL`, `Headers`, `Blob`, and `File` on hits. Cache keys still
+  use the deterministic key-hashing rules above.
+- Cache mode per run is controlled by CLI flags (see `agent-evals run --help`)
+  and by a chevron menu on each eval card in the UI.
+- The UI Stop action cancels the whole active run by terminating that run's
+  isolated execution process.
+
+## Artifacts
+
+Run output lives under `.agent-evals/runs/<run-id>/` and cache entries under
+`.agent-evals/cache/<eval-id>.json`. Files in a run directory include run
+metadata, a run summary, per-case results, and per-case trace JSON. Inspect
+these when debugging persisted output, costs, columns, traces, or failures —
+the filenames are stable even when their internal schema evolves, so pick the
+one whose name matches what you are debugging and read it directly.
+
+## Module mocking
+
+For true module replacement inside an eval, register `mock.module(...)` from
+`node:test` before dynamically importing the module graph. The CLI enables
+Node's `--experimental-test-module-mocks` flag automatically. Use dynamic
+`import(...)` inside `execute` — static imports happen too early.
+
+```ts
+import { mock } from 'node:test';
+import { defineEval } from '@ls-stack/agent-eval';
+
+defineEval({
+  id: 'module-mock-demo',
+  cases: [{ id: 'mocked-dependency', input: { customerId: 'vip-100' } }],
+  execute: async ({ input, setOutput }) => {
+    mock.module('../src/customerLookup.ts', {
+      namedExports: { lookupCustomer: async () => ({ segment: 'vip' }) },
+    });
+    const { runWorkflow } = await import('../src/workflow.ts');
+    const result = await runWorkflow(input);
+    setOutput('segment', result.segment);
+  },
+});
+```
+
+## Workflow checklist
+
+When adding or changing evals:
+
+1. Put the tracing + ambient SDK calls in the product code that runs in both
+   production and evals. Keep eval files thin.
+2. Use realistic cases drawn from real product flows; avoid placeholder inputs.
+3. `evalAssert` for hard invariants, `scores` for graded signals,
+   `passThreshold` only on scores that should gate pass/fail.
+4. Surface reviewable values through execute-context `setOutput` or ambient
+   `setEvalOutput` in shared workflow code, and shape them with `columns`
+   formats from the `ColumnFormat` type.
+5. Promote high-signal span attributes with `traceDisplay` so the UI
+   highlights them in the trace tree and detail pane.
+6. Cache costly pure spans with `cache: { key }` and pure spanless values with
+   `evalTracer.cache(...)`; never cache operations whose external side effects
+   you depend on.
+7. Sanity-check after changes: `agent-evals list`, then
+   `agent-evals run --eval <id>`. Open the UI only when you need to inspect
+   traces, trends, or fill manual scores. From an eval page, the eval actions
+   menu can copy package-manager-specific CLI run and debug commands.
+8. To debug a focused run, use
+   `agent-evals run --inspect-brk --eval <id> --case <case-id>` and attach a
+   Node.js debugger before continuing execution.

package/dist/src-BgGL7DDp.mjs

@@ -1,3 +0,0 @@
-import "./runOrchestration-H0pSUl3I.mjs";
-import "./cli-CMiCEQ-3.mjs";
-export {};