@ls-stack/agent-eval 0.12.2 → 0.13.0

package/dist/{app-D3wtTfbu.mjs → app-Dg3qYVku.mjs}

@@ -1,6 +1,6 @@
-import { C as updateManualScoreRequestSchema, S as createRunRequestSchema } from "./runOrchestration-DrgpaDaf.mjs";
-import "./src-BU6ZtVIB.mjs";
-import { t as getRunnerInstance } from "./runner-BVC9yBDu.mjs";
+import { C as updateManualScoreRequestSchema, S as createRunRequestSchema } from "./runOrchestration-H0pSUl3I.mjs";
+import "./src-Btb9RCYD.mjs";
+import { t as getRunnerInstance } from "./runner-CmbmfBG2.mjs";
 import { readFile } from "node:fs/promises";
 import { dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod/v4";

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-BEPaYHmX.mjs";
+import { t as runCli } from "./cli-lOZdhO2D.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/{cli-BEPaYHmX.mjs → cli-lOZdhO2D.mjs}

@@ -1,4 +1,4 @@
-import { A as getEvalDisplayStatus, F as runSummarySchema, J as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, dn as getEvalRegistry, f as runTouchesEval, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, j as deriveScopedSummaryFromCases, k as getEvalTitle, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, q as resolveApiCallsConfig, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-DrgpaDaf.mjs";
+import { A as getEvalDisplayStatus, F as runSummarySchema, J as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, fn as getEvalRegistry, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, j as deriveScopedSummaryFromCases, k as getEvalTitle, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, q as resolveApiCallsConfig, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-H0pSUl3I.mjs";
 import { createHash } from "node:crypto";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join, relative, resolve } from "node:path";
@@ -1050,8 +1050,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-D3wtTfbu.mjs");
-	const runnerModule = await import("./runner-BHCokR_t.mjs");
+	const appModule = await import("./app-Dg3qYVku.mjs");
+	const runnerModule = await import("./runner-BK1KX2SA.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();

package/dist/index.d.mts

@@ -2576,6 +2576,14 @@ type EvalDefinitionBase<TInput = unknown, TOutputs extends EvalOutputs = EvalOut
    * `key` is provided.
    */
   traceDisplay?: TraceDisplayInputConfig;
+  /**
+   * Whether registered background jobs should finish before outputs, tracing,
+   * and scores are finalized. Defaults to `true`.
+   *
+   * Set to `false` for evals that intentionally fire work that should not
+   * delay case finalization; late mutations are not guaranteed to persist.
+   */
+  waitForBackgroundJobs?: boolean;
   execute: (ctx: EvalExecuteContext<TInput, TOutputs>) => Promise<void> | void;
   deriveFromTracing?: (ctx: EvalDeriveContext<TInput>) => Partial<TOutputs> | Promise<Partial<TOutputs>>;
   scores?: Record<string, EvalScoreDef<TInput, TOutputs>>;
@@ -2709,7 +2717,8 @@ type EvalCaseScope = {
    * active span. Span-bound refs are appended to the owning span's
    * `cache.refs` attribute instead.
    */
-  caseCacheRefs: TraceCacheRef[];
+  caseCacheRefs: TraceCacheRef[]; /** Background promises that should settle before the case scope finalizes. */
+  pendingBackgroundJobs: Set<Promise<unknown>>;
 };
 /** Error thrown when an eval assertion fails during case execution. */
 declare class EvalAssertionError extends Error {
@@ -2724,6 +2733,14 @@ declare function getCurrentScope(): EvalCaseScope | undefined;
  * behavior without importing or inspecting the full eval scope.
  */
 declare function isInEvalScope(): boolean;
+/**
+ * Register background work that should settle before eval finalization.
+ *
+ * The original promise is returned unchanged, and its fulfillment or rejection
+ * behavior remains normal for callers. The eval runtime only waits for
+ * settlement; it does not convert background rejections into case errors.
+ */
+declare function startEvalBackgroundJob<T>(promise: Promise<T>): Promise<T>;
 /**
  * Return the authored input for the current eval case.
  *
@@ -2744,7 +2761,8 @@ declare function setScopeCacheContext(scope: EvalCaseScope, context: CacheScopeC
 type RunInEvalScopeOptions = {
   /** Authored input for the active eval case. */input?: unknown; /** Stable prefix used when generating scoped IDs with `nextEvalId()`. */
   idPrefix?: string; /** Cache adapter + mode attached to the scope before `fn` runs. */
-  cacheContext?: CacheScopeContext;
+  cacheContext?: CacheScopeContext; /** Whether registered background jobs should settle before scope finalizes. */
+  waitForBackgroundJobs?: boolean;
 };
 /**
  * Execute a callback inside a fresh eval case scope and capture its outputs,
@@ -2934,6 +2952,11 @@ type TraceSpanInfoBase = {
   kind: string;
   name: string;
   attributes?: Record<string, unknown>;
+  /**
+   * Whether this span should delay eval finalization when the returned promise
+   * is not awaited by user code. Defaults to `true`.
+   */
+  waitForBackgroundJob?: boolean;
 };
 /** Info accepted by `evalTracer.span(info, fn)` when creating an uncached span. */
 type TraceSpanInfoUncached = TraceSpanInfoBase & {
@@ -3112,4 +3135,4 @@ declare function createRunner({
  */
 declare function runCli(argv: string[]): Promise<void>;
 //#endregion
-export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as evalFreshnessStatusSchema, $t as evalAssert, A as getEvalDisplayStatus, At as traceDisplayInputConfigSchema, B as apiCallMetricPlacementSchema, Bt as jsonCellSchema, C as updateManualScoreRequestSchema, Ct as spanCacheOptionsSchema, D as extractLlmCalls, Dt as traceAttributeDisplayPlacementSchema, E as extractApiCalls, Et as traceAttributeDisplayInputSchema, F as runSummarySchema, Ft as cellValueSchema, G as llmCallMetricSchema, Gt as buildTraceTree, H as apiCallsConfigSchema, Ht as repoFileRefSchema, I as DEFAULT_API_CALLS_CONFIG, It as columnDefSchema, J as resolveLlmCallsConfig, Jt as evalTracer, K as llmCallsConfigSchema, Kt as captureEvalSpanError, L as DEFAULT_LLM_CALLS_CONFIG, Lt as columnFormatSchema, M as deriveStatusFromCaseRows, Mt as traceSpanKindSchema, N as deriveStatusFromChildStatuses, Nt as traceSpanSchema, O as getNestedAttribute, Ot as traceAttributeDisplaySchema, P as runManifestSchema, Pt as traceSpanWarningSchema, Q as caseRowSchema, Qt as appendToEvalOutput, R as agentEvalsConfigSchema, Rt as columnKindSchema, S as createRunRequestSchema, St as serializedCacheSpanSchema, T as extractCacheHits, Tt as traceAttributeDisplayFormatSchema, U as llmCallMetricFormatSchema, Ut as runArtifactRefSchema, V as apiCallMetricSchema, Vt as numberDisplayOptionsSchema, W as llmCallMetricPlacementSchema, Wt as z, X as assertionFailureSchema, Xt as hashCacheKeySync, Y as trialSelectionModeSchema, Yt as hashCacheKey, Z as caseDetailSchema, Zt as EvalAssertionError, _t as cacheModeSchema, an as nextEvalId, at as evalChartAggregateSchema, bt as cacheRecordingSchema, cn as setScopeCacheContext, ct as evalChartColorSchema, dn as getEvalRegistry, dt as evalChartTooltipExtraSchema, en as getCurrentScope, et as evalStatAggregateSchema, ft as evalChartTypeSchema, gt as cacheListItemSchema, ht as cacheFileSchema, in as mergeEvalOutput, it as scoreTraceSchema, j as deriveScopedSummaryFromCases, jt as traceSpanErrorSchema, k as getEvalTitle, kt as traceDisplayConfigSchema, ln as repoFile, lt as evalChartConfigSchema, mt as cacheEntrySchema, nn as incrementEvalOutput, nt as evalStatsConfigSchema, on as runInEvalScope, ot as evalChartAxisSchema, pt as evalChartsConfigSchema, q as resolveApiCallsConfig, qt as evalSpan, rn as isInEvalScope, rt as evalSummarySchema, sn as setEvalOutput, st as evalChartBuiltinMetricSchema, tn as getEvalCaseInput, tt as evalStatItemSchema, un as defineEval, ut as evalChartMetricSchema, vt as cacheOperationTypeSchema, w as sseEnvelopeSchema, wt as traceCacheRefSchema, xt as cacheStatusSchema, yt as cacheRecordingOpSchema, z as apiCallMetricFormatSchema, zt as fileRefSchema } from "./runOrchestration-DrgpaDaf.mjs";
-import { n as createRunner, t as runCli } from "./cli-BEPaYHmX.mjs";
-import "./src-BU6ZtVIB.mjs";
-export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+import { $ as evalFreshnessStatusSchema, $t as evalAssert, A as getEvalDisplayStatus, At as traceDisplayInputConfigSchema, B as apiCallMetricPlacementSchema, Bt as jsonCellSchema, C as updateManualScoreRequestSchema, Ct as spanCacheOptionsSchema, D as extractLlmCalls, Dt as traceAttributeDisplayPlacementSchema, E as extractApiCalls, Et as traceAttributeDisplayInputSchema, F as runSummarySchema, Ft as cellValueSchema, G as llmCallMetricSchema, Gt as buildTraceTree, H as apiCallsConfigSchema, Ht as repoFileRefSchema, I as DEFAULT_API_CALLS_CONFIG, It as columnDefSchema, J as resolveLlmCallsConfig, Jt as evalTracer, K as llmCallsConfigSchema, Kt as captureEvalSpanError, L as DEFAULT_LLM_CALLS_CONFIG, Lt as columnFormatSchema, M as deriveStatusFromCaseRows, Mt as traceSpanKindSchema, N as deriveStatusFromChildStatuses, Nt as traceSpanSchema, O as getNestedAttribute, Ot as traceAttributeDisplaySchema, P as runManifestSchema, Pt as traceSpanWarningSchema, Q as caseRowSchema, Qt as appendToEvalOutput, R as agentEvalsConfigSchema, Rt as columnKindSchema, S as createRunRequestSchema, St as serializedCacheSpanSchema, T as extractCacheHits, Tt as traceAttributeDisplayFormatSchema, U as llmCallMetricFormatSchema, Ut as runArtifactRefSchema, V as apiCallMetricSchema, Vt as numberDisplayOptionsSchema, W as llmCallMetricPlacementSchema, Wt as z, X as assertionFailureSchema, Xt as hashCacheKeySync, Y as trialSelectionModeSchema, Yt as hashCacheKey, Z as caseDetailSchema, Zt as EvalAssertionError, _t as cacheModeSchema, an as nextEvalId, at as evalChartAggregateSchema, bt as cacheRecordingSchema, cn as setScopeCacheContext, ct as evalChartColorSchema, dn as defineEval, dt as evalChartTooltipExtraSchema, en as getCurrentScope, et as evalStatAggregateSchema, fn as getEvalRegistry, ft as evalChartTypeSchema, gt as cacheListItemSchema, ht as cacheFileSchema, in as mergeEvalOutput, it as scoreTraceSchema, j as deriveScopedSummaryFromCases, jt as traceSpanErrorSchema, k as getEvalTitle, kt as traceDisplayConfigSchema, ln as startEvalBackgroundJob, lt as evalChartConfigSchema, mt as cacheEntrySchema, nn as incrementEvalOutput, nt as evalStatsConfigSchema, on as runInEvalScope, ot as evalChartAxisSchema, pt as evalChartsConfigSchema, q as resolveApiCallsConfig, qt as evalSpan, rn as isInEvalScope, rt as evalSummarySchema, sn as setEvalOutput, st as evalChartBuiltinMetricSchema, tn as getEvalCaseInput, tt as evalStatItemSchema, un as repoFile, ut as evalChartMetricSchema, vt as cacheOperationTypeSchema, w as sseEnvelopeSchema, wt as traceCacheRefSchema, xt as cacheStatusSchema, yt as cacheRecordingOpSchema, z as apiCallMetricFormatSchema, zt as fileRefSchema } from "./runOrchestration-H0pSUl3I.mjs";
+import { n as createRunner, t as runCli } from "./cli-lOZdhO2D.mjs";
+import "./src-Btb9RCYD.mjs";
+export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.mjs

@@ -1,4 +1,4 @@
-import { F as runSummarySchema, It as columnDefSchema, P as runManifestSchema, S as createRunRequestSchema, nt as evalStatsConfigSchema, pt as evalChartsConfigSchema, t as executeRun, v as loadConfig, x as createFsCacheStore } from "./runOrchestration-DrgpaDaf.mjs";
+import { F as runSummarySchema, It as columnDefSchema, P as runManifestSchema, S as createRunRequestSchema, nt as evalStatsConfigSchema, pt as evalChartsConfigSchema, t as executeRun, v as loadConfig, x as createFsCacheStore } from "./runOrchestration-H0pSUl3I.mjs";
 import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { z } from "zod/v4";

package/dist/{runOrchestration-DrgpaDaf.mjs → runOrchestration-H0pSUl3I.mjs}

@@ -68,6 +68,30 @@ function getCurrentScope() {
 function isInEvalScope() {
 	return getCurrentScope() !== void 0;
 }
+function registerBackgroundJobInScope(scope, promise) {
+	const trackedPromise = promise.then(() => {
+		scope.pendingBackgroundJobs.delete(trackedPromise);
+	}, () => {
+		scope.pendingBackgroundJobs.delete(trackedPromise);
+	});
+	scope.pendingBackgroundJobs.add(trackedPromise);
+	return promise;
+}
+async function drainBackgroundJobs(scope) {
+	while (scope.pendingBackgroundJobs.size > 0) await Promise.allSettled([...scope.pendingBackgroundJobs]);
+}
+/**
+* Register background work that should settle before eval finalization.
+*
+* The original promise is returned unchanged, and its fulfillment or rejection
+* behavior remains normal for callers. The eval runtime only waits for
+* settlement; it does not convert background rejections into case errors.
+*/
+function startEvalBackgroundJob(promise) {
+	const scope = getCurrentScope();
+	if (!scope) return promise;
+	return registerBackgroundJobInScope(scope, promise);
+}
 function isObjectLike(value) {
 	return typeof value === "object" && value !== null;
 }
@@ -117,18 +141,22 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		recordingStack: [],
 		replayingDepth: 0,
 		cacheContext: options.cacheContext,
-		caseCacheRefs: []
+		caseCacheRefs: [],
+		pendingBackgroundJobs: /* @__PURE__ */ new Set()
 	};
 	activeEvalScopeCount++;
 	try {
 		return await scopeStorage.run(scope, async () => {
 			try {
+				const result = await fn();
+				if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
 				return {
-					result: await fn(),
+					result,
 					scope,
 					error: void 0
 				};
 			} catch (error) {
+				if (options.waitForBackgroundJobs !== false) await drainBackgroundJobs(scope);
 				return {
 					result: void 0,
 					scope,
@@ -2849,7 +2877,12 @@ function captureEvalSpanError(errorOrErrors, ...additionalErrorsOrOptions) {
 		appendSpanErrors(currentSpan, errors);
 	});
 }
-async function traceSpan(info, fn) {
+function traceSpan(info, fn) {
+	const promise = traceSpanInternal(info, fn);
+	if (!getCurrentScope() || info.waitForBackgroundJob === false) return promise;
+	return startEvalBackgroundJob(promise);
+}
+async function traceSpanInternal(info, fn) {
 	const scope = getCurrentScope();
 	if (!scope) return await fn(noopActiveSpan());
 	const id = generateSpanId();
@@ -5432,6 +5465,7 @@ async function runCase(params) {
 	}, {
 		input: evalCase.input,
 		idPrefix: scopedIdPrefix,
+		waitForBackgroundJobs: evalDef.waitForBackgroundJobs !== false,
 		cacheContext: cacheAdapter ? {
 			adapter: cacheAdapter,
 			mode: cacheMode,
@@ -5961,4 +5995,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { evalFreshnessStatusSchema as $, evalAssert as $t, getEvalDisplayStatus as A, traceDisplayInputConfigSchema as At, apiCallMetricPlacementSchema as B, jsonCellSchema as Bt, updateManualScoreRequestSchema as C, spanCacheOptionsSchema as Ct, extractLlmCalls as D, traceAttributeDisplayPlacementSchema as Dt, extractApiCalls as E, traceAttributeDisplayInputSchema as Et, runSummarySchema as F, cellValueSchema as Ft, llmCallMetricSchema as G, buildTraceTree as Gt, apiCallsConfigSchema as H, repoFileRefSchema as Ht, DEFAULT_API_CALLS_CONFIG as I, columnDefSchema as It, resolveLlmCallsConfig as J, evalTracer as Jt, llmCallsConfigSchema as K, captureEvalSpanError as Kt, DEFAULT_LLM_CALLS_CONFIG as L, columnFormatSchema as Lt, deriveStatusFromCaseRows as M, traceSpanKindSchema as Mt, deriveStatusFromChildStatuses as N, traceSpanSchema as Nt, getNestedAttribute as O, traceAttributeDisplaySchema as Ot, runManifestSchema as P, traceSpanWarningSchema as Pt, caseRowSchema as Q, appendToEvalOutput as Qt, agentEvalsConfigSchema as R, columnKindSchema as Rt, createRunRequestSchema as S, serializedCacheSpanSchema as St, extractCacheHits as T, traceAttributeDisplayFormatSchema as Tt, llmCallMetricFormatSchema as U, runArtifactRefSchema as Ut, apiCallMetricSchema as V, numberDisplayOptionsSchema as Vt, llmCallMetricPlacementSchema as W, z$1 as Wt, assertionFailureSchema as X, hashCacheKeySync as Xt, trialSelectionModeSchema as Y, hashCacheKey as Yt, caseDetailSchema as Z, EvalAssertionError as Zt, loadEvalModule as _, cacheModeSchema as _t, loadPersistedRunSnapshot as a, nextEvalId as an, evalChartAggregateSchema as at, normalizeScoreDef as b, cacheRecordingSchema as bt, persistCaseDetail as c, setScopeCacheContext as cn, evalChartColorSchema as ct, recomputePersistedCaseStatus as d, getEvalRegistry as dn, evalChartTooltipExtraSchema as dt, getCurrentScope as en, evalStatAggregateSchema as et, runTouchesEval as f, evalChartTypeSchema as ft, setLatestRunInfoMap as g, cacheListItemSchema as gt, getTargetEvalIds as h, cacheFileSchema as ht, getLatestRunInfos as i, mergeEvalOutput as in, scoreTraceSchema as it, deriveScopedSummaryFromCases as j, traceSpanErrorSchema as jt, getEvalTitle as k, traceDisplayConfigSchema as kt, persistRunState as l, repoFile as ln, evalChartConfigSchema as lt, buildEvalSummary as m, cacheEntrySchema as mt, generateRunId as n, incrementEvalOutput as nn, evalStatsConfigSchema as nt, loadPersistedRunSnapshots as o, runInEvalScope as on, evalChartAxisSchema as ot, resolveArtifactPath as p, evalChartsConfigSchema as pt, resolveApiCallsConfig as q, evalSpan as qt, getLastRunStatuses as r, isInEvalScope as rn, evalSummarySchema as rt, nextShortIdFromSnapshots as s, setEvalOutput as sn, evalChartBuiltinMetricSchema as st, executeRun as t, getEvalCaseInput as tn, evalStatItemSchema as tt, recomputeEvalStatusesInRuns as u, defineEval as un, evalChartMetricSchema as ut, loadConfig as v, cacheOperationTypeSchema as vt, sseEnvelopeSchema as w, traceCacheRefSchema as wt, createFsCacheStore as x, cacheStatusSchema as xt, buildDeclaredColumnDefs as y, cacheRecordingOpSchema as yt, apiCallMetricFormatSchema as z, fileRefSchema as zt };
+export { evalFreshnessStatusSchema as $, evalAssert as $t, getEvalDisplayStatus as A, traceDisplayInputConfigSchema as At, apiCallMetricPlacementSchema as B, jsonCellSchema as Bt, updateManualScoreRequestSchema as C, spanCacheOptionsSchema as Ct, extractLlmCalls as D, traceAttributeDisplayPlacementSchema as Dt, extractApiCalls as E, traceAttributeDisplayInputSchema as Et, runSummarySchema as F, cellValueSchema as Ft, llmCallMetricSchema as G, buildTraceTree as Gt, apiCallsConfigSchema as H, repoFileRefSchema as Ht, DEFAULT_API_CALLS_CONFIG as I, columnDefSchema as It, resolveLlmCallsConfig as J, evalTracer as Jt, llmCallsConfigSchema as K, captureEvalSpanError as Kt, DEFAULT_LLM_CALLS_CONFIG as L, columnFormatSchema as Lt, deriveStatusFromCaseRows as M, traceSpanKindSchema as Mt, deriveStatusFromChildStatuses as N, traceSpanSchema as Nt, getNestedAttribute as O, traceAttributeDisplaySchema as Ot, runManifestSchema as P, traceSpanWarningSchema as Pt, caseRowSchema as Q, appendToEvalOutput as Qt, agentEvalsConfigSchema as R, columnKindSchema as Rt, createRunRequestSchema as S, serializedCacheSpanSchema as St, extractCacheHits as T, traceAttributeDisplayFormatSchema as Tt, llmCallMetricFormatSchema as U, runArtifactRefSchema as Ut, apiCallMetricSchema as V, numberDisplayOptionsSchema as Vt, llmCallMetricPlacementSchema as W, z$1 as Wt, assertionFailureSchema as X, hashCacheKeySync as Xt, trialSelectionModeSchema as Y, hashCacheKey as Yt, caseDetailSchema as Z, EvalAssertionError as Zt, loadEvalModule as _, cacheModeSchema as _t, loadPersistedRunSnapshot as a, nextEvalId as an, evalChartAggregateSchema as at, normalizeScoreDef as b, cacheRecordingSchema as bt, persistCaseDetail as c, setScopeCacheContext as cn, evalChartColorSchema as ct, recomputePersistedCaseStatus as d, defineEval as dn, evalChartTooltipExtraSchema as dt, getCurrentScope as en, evalStatAggregateSchema as et, runTouchesEval as f, getEvalRegistry as fn, evalChartTypeSchema as ft, setLatestRunInfoMap as g, cacheListItemSchema as gt, getTargetEvalIds as h, cacheFileSchema as ht, getLatestRunInfos as i, mergeEvalOutput as in, scoreTraceSchema as it, deriveScopedSummaryFromCases as j, traceSpanErrorSchema as jt, getEvalTitle as k, traceDisplayConfigSchema as kt, persistRunState as l, startEvalBackgroundJob as ln, evalChartConfigSchema as lt, buildEvalSummary as m, cacheEntrySchema as mt, generateRunId as n, incrementEvalOutput as nn, evalStatsConfigSchema as nt, loadPersistedRunSnapshots as o, runInEvalScope as on, evalChartAxisSchema as ot, resolveArtifactPath as p, evalChartsConfigSchema as pt, resolveApiCallsConfig as q, evalSpan as qt, getLastRunStatuses as r, isInEvalScope as rn, evalSummarySchema as rt, nextShortIdFromSnapshots as s, setEvalOutput as sn, evalChartBuiltinMetricSchema as st, executeRun as t, getEvalCaseInput as tn, evalStatItemSchema as tt, recomputeEvalStatusesInRuns as u, repoFile as un, evalChartMetricSchema as ut, loadConfig as v, cacheOperationTypeSchema as vt, sseEnvelopeSchema as w, traceCacheRefSchema as wt, createFsCacheStore as x, cacheStatusSchema as xt, buildDeclaredColumnDefs as y, cacheRecordingOpSchema as yt, apiCallMetricFormatSchema as z, fileRefSchema as zt };

package/dist/{runner-BHCokR_t.mjs → runner-BK1KX2SA.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-BVC9yBDu.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-CmbmfBG2.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-BVC9yBDu.mjs → runner-CmbmfBG2.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-BEPaYHmX.mjs";
-import "./src-BU6ZtVIB.mjs";
+import { n as createRunner } from "./cli-lOZdhO2D.mjs";
+import "./src-Btb9RCYD.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/src-Btb9RCYD.mjs

@@ -0,0 +1,3 @@
+import "./runOrchestration-H0pSUl3I.mjs";
+import "./cli-lOZdhO2D.mjs";
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.12.2",
+  "version": "0.13.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"

package/dist/src-BU6ZtVIB.mjs

@@ -1,3 +0,0 @@
-import "./runOrchestration-DrgpaDaf.mjs";
-import "./cli-BEPaYHmX.mjs";
-export {};