@ls-stack/agent-eval 0.9.0 → 0.10.0

package/dist/{app-hkNNN_jn.mjs → app-sK9CjpNI.mjs}

@@ -1,6 +1,6 @@
-import { C as updateManualScoreRequestSchema, S as createRunRequestSchema } from "./runOrchestration-DA4Rh5g0.mjs";
-import "./src-CfprG1RW.mjs";
-import { t as getRunnerInstance } from "./runner-DTP5Ui4_.mjs";
+import { C as updateManualScoreRequestSchema, S as createRunRequestSchema } from "./runOrchestration-DwqX9_T7.mjs";
+import "./src-Bx-CV6Wo.mjs";
+import { t as getRunnerInstance } from "./runner-Gtlmvm3w.mjs";
 import { readFile } from "node:fs/promises";
 import { dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod/v4";

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-DrPk66xh.mjs";
+import { t as runCli } from "./cli-DLlRkyLH.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/{cli-DrPk66xh.mjs → cli-DLlRkyLH.mjs}

@@ -1,4 +1,4 @@
-import { A as deriveScopedSummaryFromCases, O as getEvalTitle, P as runSummarySchema, V as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, k as getEvalDisplayStatus, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, r as getLastRunStatuses, rn as getEvalRegistry, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-DA4Rh5g0.mjs";
+import { A as deriveScopedSummaryFromCases, O as getEvalTitle, P as runSummarySchema, V as resolveLlmCallsConfig, _ as loadEvalModule, a as loadPersistedRunSnapshot, b as normalizeScoreDef, c as persistCaseDetail, d as recomputePersistedCaseStatus, f as runTouchesEval, g as setLatestRunInfoMap, h as getTargetEvalIds, i as getLatestRunInfos, in as getEvalRegistry, k as getEvalDisplayStatus, l as persistRunState, m as buildEvalSummary, n as generateRunId, o as loadPersistedRunSnapshots, p as resolveArtifactPath, r as getLastRunStatuses, s as nextShortIdFromSnapshots, u as recomputeEvalStatusesInRuns, v as loadConfig, x as createFsCacheStore, y as buildDeclaredColumnDefs } from "./runOrchestration-DwqX9_T7.mjs";
 import { createHash } from "node:crypto";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join, relative, resolve } from "node:path";
@@ -1045,8 +1045,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-hkNNN_jn.mjs");
-	const runnerModule = await import("./runner-BzT3B9OF.mjs");
+	const appModule = await import("./app-sK9CjpNI.mjs");
+	const runnerModule = await import("./runner-JrBz8ISs.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();

package/dist/index.d.mts

@@ -2196,7 +2196,7 @@ type EvalCase<TInput> = {
   input: TInput;
   tags?: string[];
 };
-/** Runtime output values collected from `setEvalOutput` and `deriveFromTracing`. */
+/** Runtime output values collected from output helpers and `deriveFromTracing`. */
 type EvalOutputs = Record<string, unknown>;
 /**
  * Schema used to validate and type an eval's collected runtime outputs.
@@ -2249,9 +2249,31 @@ type EvalTraceTree = {
   flattenDfs: () => EvalTraceSpan[];
   checkpoints: Map<string, unknown>;
 };
+/** Type-safe output writer passed to an eval's `execute` function. */
+type EvalSetOutput<TOutputs extends EvalOutputs = EvalOutputs> = <TKey extends Extract<keyof TOutputs, string>>(
+/**
+ * Output field to record. For narrowed output maps, this must be one of the
+ * known output keys.
+ */
+
+key: TKey,
+/**
+ * Value for the output field. For narrowed output maps, this must match the
+ * field's declared output type.
+ */
+
+value: TOutputs[TKey]) => void;
 /** Context passed to an eval's `execute` function for a single case run. */
-type EvalExecuteContext<TInput> = {
-  input: TInput;
+type EvalExecuteContext<TInput, TOutputs extends EvalOutputs = EvalOutputs> = {
+  /** Authored input for the active eval case. */input: TInput;
+  /**
+   * Record or replace an output value for the current case scope.
+   *
+   * When the eval has a narrowed outputs generic, keys and values are typed
+   * from that output map. The recorded values are still validated by
+   * `outputsSchema` before computed scores run.
+   */
+  setOutput: EvalSetOutput<TOutputs>;
 };
 /** Context passed to `deriveFromTracing` after execution has completed. */
 type EvalDeriveContext<TInput> = {
@@ -2293,8 +2315,31 @@ type EvalManualScoreDef = EvalColumnOverride & {
    */
   passThreshold?: number;
 };
-/** Complete authored eval definition consumed by `defineEval`. */
-type EvalDefinition<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs> = {
+type EvalDefinitionOutputSchemaConfig<TOutputs extends EvalOutputs> = [EvalOutputs] extends [TOutputs] ? {
+  /**
+   * Optional schema for runtime outputs collected through output helpers
+   * and `deriveFromTracing`.
+   *
+   * The runner validates configured output fields before scoring. For
+   * Zod object schemas, only declared keys are passed to the schema;
+   * parsed fields are merged back into the raw output map, so schema
+   * defaults and transforms apply to configured fields while
+   * unconfigured outputs are kept unchanged. Validation failures mark
+   * the case as failed and skip computed scores.
+   */
+  outputsSchema?: EvalOutputsSchema<TOutputs>;
+} : {
+  /**
+   * Required schema for typed runtime outputs collected through output
+   * helpers and `deriveFromTracing`.
+   *
+   * When `EvalDefinition` or `defineEval` receives an explicit narrowed
+   * outputs generic, this schema is required so scorer inputs are backed
+   * by runtime validation before computed scores run.
+   */
+  outputsSchema: EvalOutputsSchema<TOutputs>;
+};
+type EvalDefinitionBase<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs> = {
   id: string;
   title?: string;
   /**
@@ -2304,17 +2349,6 @@ type EvalDefinition<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs
    * eval once using a synthetic case with empty object input.
    */
   cases?: EvalCase<TInput>[] | (() => Promise<EvalCase<TInput>[]>);
-  /**
-   * Optional schema for runtime outputs collected through `setEvalOutput` and
-   * `deriveFromTracing`.
-   *
-   * The runner validates configured output fields before scoring. For Zod
-   * object schemas, only declared keys are passed to the schema; parsed fields
-   * are merged back into the raw output map, so schema defaults and transforms
-   * apply to configured fields while unconfigured outputs are kept unchanged.
-   * Validation failures mark the case as failed and skip computed scores.
-   */
-  outputsSchema?: EvalOutputsSchema<TOutputs>;
   columns?: EvalColumns;
   /**
    * Per-eval trace attribute display rules for the UI.
@@ -2324,7 +2358,7 @@ type EvalDefinition<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs
    * `key` is provided.
    */
   traceDisplay?: TraceDisplayInputConfig;
-  execute: (ctx: EvalExecuteContext<TInput>) => Promise<void> | void;
+  execute: (ctx: EvalExecuteContext<TInput, TOutputs>) => Promise<void> | void;
   deriveFromTracing?: (ctx: EvalDeriveContext<TInput>) => Partial<TOutputs> | Promise<Partial<TOutputs>>;
   scores?: Record<string, EvalScoreDef<TInput, TOutputs>>;
   /**
@@ -2359,13 +2393,21 @@ type EvalDefinition<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs
    *
    * Each chart declares its `type` (`area | line | bar`) and one or more
    * `metrics`. Built-in metrics (`passRate`, `durationMs`) aggregate
-   * the run summary. Column metrics aggregate a score or numeric `setEvalOutput`
-   * column across the run using an `aggregate` reducer (`avg`, `sum`, `min`,
-   * `max`, `latest`, `passThresholdRate`). `passThresholdRate` requires a
-   * score column with `passThreshold`.
+   * the run summary. Column metrics aggregate a score or numeric output column
+   * across the run using an `aggregate` reducer (`avg`, `sum`, `min`, `max`,
+   * `latest`, `passThresholdRate`). `passThresholdRate` requires a score column
+   * with `passThreshold`.
    */
   charts?: EvalChartsConfig;
 };
+/**
+ * Complete authored eval definition consumed by `defineEval`.
+ *
+ * `outputsSchema` is optional for the default loose output map. When the
+ * `TOutputs` generic is narrowed, `outputsSchema` is required so the runtime
+ * validates collected outputs before exposing them as typed scorer inputs.
+ */
+type EvalDefinition<TInput = unknown, TOutputs extends EvalOutputs = EvalOutputs> = EvalDefinitionBase<TInput, TOutputs> & EvalDefinitionOutputSchemaConfig<TOutputs>;
 //#endregion
 //#region ../sdk/src/defineEval.d.ts
 /**
@@ -2423,7 +2465,9 @@ type CacheRecordingFrame = {
 };
 /** Mutable per-case runtime state stored in async local storage. */
 type EvalCaseScope = {
-  caseId: string; /** Authored input for the current case, when provided by the runner. */
+  caseId: string; /** Stable prefix used by `nextEvalId()` for this eval case scope. */
+  idPrefix: string | undefined; /** Monotonic per-scope counter used by `nextEvalId()`. */
+  nextEvalIdCounter: number; /** Authored input for the current case, when provided by the runner. */
   input?: unknown;
   outputs: Record<string, unknown>; /** Structured assertion failures recorded for the current case. */
   assertionFailures: AssertionFailure[];
@@ -2480,7 +2524,8 @@ declare function getEvalCaseInput(path: string): unknown;
 declare function setScopeCacheContext(scope: EvalCaseScope, context: CacheScopeContext): void;
 /** Optional inputs accepted when starting a new eval case scope. */
 type RunInEvalScopeOptions = {
-  /** Authored input for the active eval case. */input?: unknown; /** Cache adapter + mode attached to the scope before `fn` runs. */
+  /** Authored input for the active eval case. */input?: unknown; /** Stable prefix used when generating scoped IDs with `nextEvalId()`. */
+  idPrefix?: string; /** Cache adapter + mode attached to the scope before `fn` runs. */
   cacheContext?: CacheScopeContext;
 };
 /**
@@ -2492,6 +2537,15 @@ declare function runInEvalScope<T>(caseId: string, fn: () => Promise<T> | T, opt
   scope: EvalCaseScope;
   error: Error | undefined;
 }>;
+/**
+ * Return the next deterministic ID for the active eval case execution.
+ *
+ * The runner derives the ID prefix from the eval file, eval id, and case id,
+ * then this helper appends a per-scope sequence number. Calls outside an
+ * active eval case scope throw so accidental product-code usage is caught
+ * immediately.
+ */
+declare function nextEvalId(): string;
 /**
  * Record or replace an output value for the current case scope.
  *
@@ -2833,4 +2887,4 @@ declare function createRunner({
  */
 declare function runCli(argv: string[]): Promise<void>;
 //#endregion
-export { type AgentEvalsConfig, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+export { type AgentEvalsConfig, type AssertionFailure, type CacheAdapter, type CacheEntry, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_LLM_CALLS_CONFIG, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallsConfigInput, type NumberDisplayOptions, type RepoFileRef, type ResolvedLlmCallMetric, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as evalChartAxisSchema, $t as setEvalOutput, A as deriveScopedSummaryFromCases, At as columnFormatSchema, B as llmCallsConfigSchema, Bt as evalSpan, C as updateManualScoreRequestSchema, Ct as traceDisplayInputConfigSchema, D as getNestedAttribute, Dt as traceSpanWarningSchema, E as extractLlmCalls, Et as traceSpanSchema, F as DEFAULT_LLM_CALLS_CONFIG, Ft as repoFileRefSchema, G as caseRowSchema, Gt as appendToEvalOutput, H as trialSelectionModeSchema, Ht as hashCacheKey, I as agentEvalsConfigSchema, It as runArtifactRefSchema, J as evalStatItemSchema, Jt as getEvalCaseInput, K as evalFreshnessStatusSchema, Kt as evalAssert, L as llmCallMetricFormatSchema, Lt as z, M as deriveStatusFromChildStatuses, Mt as fileRefSchema, N as runManifestSchema, Nt as jsonCellSchema, O as getEvalTitle, Ot as cellValueSchema, P as runSummarySchema, Pt as numberDisplayOptionsSchema, Q as evalChartAggregateSchema, Qt as runInEvalScope, R as llmCallMetricPlacementSchema, Rt as buildTraceTree, S as createRunRequestSchema, St as traceDisplayConfigSchema, T as extractCacheHits, Tt as traceSpanKindSchema, U as assertionFailureSchema, Ut as hashCacheKeySync, V as resolveLlmCallsConfig, Vt as evalTracer, W as caseDetailSchema, Wt as EvalAssertionError, X as evalSummarySchema, Xt as isInEvalScope, Y as evalStatsConfigSchema, Yt as incrementEvalOutput, Z as scoreTraceSchema, Zt as mergeEvalOutput, _t as traceCacheRefSchema, at as evalChartTypeSchema, bt as traceAttributeDisplayPlacementSchema, ct as cacheFileSchema, dt as cacheOperationTypeSchema, en as setScopeCacheContext, et as evalChartBuiltinMetricSchema, ft as cacheRecordingOpSchema, gt as spanCacheOptionsSchema, ht as serializedCacheSpanSchema, it as evalChartTooltipExtraSchema, j as deriveStatusFromCaseRows, jt as columnKindSchema, k as getEvalDisplayStatus, kt as columnDefSchema, lt as cacheListItemSchema, mt as cacheStatusSchema, nn as defineEval, nt as evalChartConfigSchema, ot as evalChartsConfigSchema, pt as cacheRecordingSchema, q as evalStatAggregateSchema, qt as getCurrentScope, rn as getEvalRegistry, rt as evalChartMetricSchema, st as cacheEntrySchema, tn as repoFile, tt as evalChartColorSchema, ut as cacheModeSchema, vt as traceAttributeDisplayFormatSchema, w as sseEnvelopeSchema, wt as traceSpanErrorSchema, xt as traceAttributeDisplaySchema, yt as traceAttributeDisplayInputSchema, z as llmCallMetricSchema, zt as captureEvalSpanError } from "./runOrchestration-DA4Rh5g0.mjs";
-import { n as createRunner, t as runCli } from "./cli-DrPk66xh.mjs";
-import "./src-CfprG1RW.mjs";
-export { DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+import { $ as evalChartAxisSchema, $t as runInEvalScope, A as deriveScopedSummaryFromCases, At as columnFormatSchema, B as llmCallsConfigSchema, Bt as evalSpan, C as updateManualScoreRequestSchema, Ct as traceDisplayInputConfigSchema, D as getNestedAttribute, Dt as traceSpanWarningSchema, E as extractLlmCalls, Et as traceSpanSchema, F as DEFAULT_LLM_CALLS_CONFIG, Ft as repoFileRefSchema, G as caseRowSchema, Gt as appendToEvalOutput, H as trialSelectionModeSchema, Ht as hashCacheKey, I as agentEvalsConfigSchema, It as runArtifactRefSchema, J as evalStatItemSchema, Jt as getEvalCaseInput, K as evalFreshnessStatusSchema, Kt as evalAssert, L as llmCallMetricFormatSchema, Lt as z, M as deriveStatusFromChildStatuses, Mt as fileRefSchema, N as runManifestSchema, Nt as jsonCellSchema, O as getEvalTitle, Ot as cellValueSchema, P as runSummarySchema, Pt as numberDisplayOptionsSchema, Q as evalChartAggregateSchema, Qt as nextEvalId, R as llmCallMetricPlacementSchema, Rt as buildTraceTree, S as createRunRequestSchema, St as traceDisplayConfigSchema, T as extractCacheHits, Tt as traceSpanKindSchema, U as assertionFailureSchema, Ut as hashCacheKeySync, V as resolveLlmCallsConfig, Vt as evalTracer, W as caseDetailSchema, Wt as EvalAssertionError, X as evalSummarySchema, Xt as isInEvalScope, Y as evalStatsConfigSchema, Yt as incrementEvalOutput, Z as scoreTraceSchema, Zt as mergeEvalOutput, _t as traceCacheRefSchema, at as evalChartTypeSchema, bt as traceAttributeDisplayPlacementSchema, ct as cacheFileSchema, dt as cacheOperationTypeSchema, en as setEvalOutput, et as evalChartBuiltinMetricSchema, ft as cacheRecordingOpSchema, gt as spanCacheOptionsSchema, ht as serializedCacheSpanSchema, in as getEvalRegistry, it as evalChartTooltipExtraSchema, j as deriveStatusFromCaseRows, jt as columnKindSchema, k as getEvalDisplayStatus, kt as columnDefSchema, lt as cacheListItemSchema, mt as cacheStatusSchema, nn as repoFile, nt as evalChartConfigSchema, ot as evalChartsConfigSchema, pt as cacheRecordingSchema, q as evalStatAggregateSchema, qt as getCurrentScope, rn as defineEval, rt as evalChartMetricSchema, st as cacheEntrySchema, tn as setScopeCacheContext, tt as evalChartColorSchema, ut as cacheModeSchema, vt as traceAttributeDisplayFormatSchema, w as sseEnvelopeSchema, wt as traceSpanErrorSchema, xt as traceAttributeDisplaySchema, yt as traceAttributeDisplayInputSchema, z as llmCallMetricSchema, zt as captureEvalSpanError } from "./runOrchestration-DwqX9_T7.mjs";
+import { n as createRunner, t as runCli } from "./cli-DLlRkyLH.mjs";
+import "./src-Bx-CV6Wo.mjs";
+export { DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.mjs

@@ -1,4 +1,4 @@
-import { N as runManifestSchema, P as runSummarySchema, S as createRunRequestSchema, Y as evalStatsConfigSchema, kt as columnDefSchema, ot as evalChartsConfigSchema, t as executeRun, v as loadConfig, x as createFsCacheStore } from "./runOrchestration-DA4Rh5g0.mjs";
+import { N as runManifestSchema, P as runSummarySchema, S as createRunRequestSchema, Y as evalStatsConfigSchema, kt as columnDefSchema, ot as evalChartsConfigSchema, t as executeRun, v as loadConfig, x as createFsCacheStore } from "./runOrchestration-DwqX9_T7.mjs";
 import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { z } from "zod/v4";

package/dist/{runOrchestration-DA4Rh5g0.mjs → runOrchestration-DwqX9_T7.mjs}

@@ -105,6 +105,8 @@ function setScopeCacheContext(scope, context) {
 async function runInEvalScope(caseId, fn, options = {}) {
 	const scope = {
 		caseId,
+		idPrefix: options.idPrefix,
+		nextEvalIdCounter: 0,
 		input: options.input,
 		outputs: {},
 		assertionFailures: [],
@@ -138,6 +140,21 @@ async function runInEvalScope(caseId, fn, options = {}) {
 		activeEvalScopeCount--;
 	}
 }
+/**
+* Return the next deterministic ID for the active eval case execution.
+*
+* The runner derives the ID prefix from the eval file, eval id, and case id,
+* then this helper appends a per-scope sequence number. Calls outside an
+* active eval case scope throw so accidental product-code usage is caught
+* immediately.
+*/
+function nextEvalId() {
+	const scope = getCurrentScope();
+	if (!scope) throw new Error("nextEvalId() must be called inside an active eval case");
+	if (scope.idPrefix === void 0) throw new Error("nextEvalId() requires a runner-provided eval id prefix");
+	scope.nextEvalIdCounter++;
+	return `${scope.idPrefix}-${scope.nextEvalIdCounter}`;
+}
 function recordOpIfActive(scope, op) {
 	if (scope.replayingDepth > 0) return;
 	const top = scope.recordingStack.at(-1);
@@ -5186,14 +5203,35 @@ function resolveRunnableEvalCases(params) {
 		input: {}
 	}];
 }
+function toStableIdSegment(value) {
+	const segment = value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+	return segment.length > 0 ? segment : "id";
+}
+function buildScopedEvalIdPrefix(params) {
+	const fileIdentity = relative(params.workspaceRoot, params.evalFilePath).replaceAll("\\", "/");
+	return [
+		toStableIdSegment(params.evalId),
+		toStableIdSegment(fileIdentity),
+		toStableIdSegment(params.caseId)
+	].join("-");
+}
 async function callWithUnknownResult(fn, args) {
 	return await Reflect.apply(fn, void 0, args);
 }
 async function runCase(params) {
-	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, startTime, cacheAdapter, cacheMode, codeFingerprint, moduleIsolation, artifactDir, runId } = params;
+	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, startTime, cacheAdapter, cacheMode, codeFingerprint, moduleIsolation, evalFilePath, workspaceRoot, artifactDir, runId } = params;
+	const scopedIdPrefix = buildScopedEvalIdPrefix({
+		evalId,
+		evalFilePath,
+		caseId: evalCase.id,
+		workspaceRoot
+	});
 	const { scope, error: executeError } = await runInEvalScope(evalCase.id, async () => {
 		const execute = async () => {
-			await Reflect.apply(evalDef.execute, evalDef, [{ input: evalCase.input }]);
+			await Reflect.apply(evalDef.execute, evalDef, [{
+				input: evalCase.input,
+				setOutput: setEvalOutput
+			}]);
 		};
 		if (moduleIsolation === void 0) {
 			await execute();
@@ -5202,6 +5240,7 @@ async function runCase(params) {
 		await runWithModuleIsolation(moduleIsolation, execute);
 	}, {
 		input: evalCase.input,
+		idPrefix: scopedIdPrefix,
 		cacheContext: cacheAdapter ? {
 			adapter: cacheAdapter,
 			mode: cacheMode,
@@ -5246,6 +5285,7 @@ async function runCase(params) {
 			return await runWithModuleIsolation(moduleIsolation, computeScore);
 		}, {
 			input: evalCase.input,
+			idPrefix: `${scopedIdPrefix}-score-${toStableIdSegment(key)}`,
 			cacheContext: cacheAdapter ? {
 				adapter: cacheAdapter,
 				mode: cacheMode,
@@ -5573,6 +5613,8 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 											cacheMode,
 											codeFingerprint,
 											moduleIsolation,
+											evalFilePath,
+											workspaceRoot,
 											artifactDir: join(runDir, "artifacts"),
 											runId: runState.manifest.id
 										});
@@ -5714,4 +5756,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { evalChartAxisSchema as $, setEvalOutput as $t, deriveScopedSummaryFromCases as A, columnFormatSchema as At, llmCallsConfigSchema as B, evalSpan as Bt, updateManualScoreRequestSchema as C, traceDisplayInputConfigSchema as Ct, getNestedAttribute as D, traceSpanWarningSchema as Dt, extractLlmCalls as E, traceSpanSchema as Et, DEFAULT_LLM_CALLS_CONFIG as F, repoFileRefSchema as Ft, caseRowSchema as G, appendToEvalOutput as Gt, trialSelectionModeSchema as H, hashCacheKey as Ht, agentEvalsConfigSchema as I, runArtifactRefSchema as It, evalStatItemSchema as J, getEvalCaseInput as Jt, evalFreshnessStatusSchema as K, evalAssert as Kt, llmCallMetricFormatSchema as L, z$1 as Lt, deriveStatusFromChildStatuses as M, fileRefSchema as Mt, runManifestSchema as N, jsonCellSchema as Nt, getEvalTitle as O, cellValueSchema as Ot, runSummarySchema as P, numberDisplayOptionsSchema as Pt, evalChartAggregateSchema as Q, runInEvalScope as Qt, llmCallMetricPlacementSchema as R, buildTraceTree as Rt, createRunRequestSchema as S, traceDisplayConfigSchema as St, extractCacheHits as T, traceSpanKindSchema as Tt, assertionFailureSchema as U, hashCacheKeySync as Ut, resolveLlmCallsConfig as V, evalTracer as Vt, caseDetailSchema as W, EvalAssertionError as Wt, evalSummarySchema as X, isInEvalScope as Xt, evalStatsConfigSchema as Y, incrementEvalOutput as Yt, scoreTraceSchema as Z, mergeEvalOutput as Zt, loadEvalModule as _, traceCacheRefSchema as _t, loadPersistedRunSnapshot as a, evalChartTypeSchema as at, normalizeScoreDef as b, traceAttributeDisplayPlacementSchema as bt, persistCaseDetail as c, cacheFileSchema as ct, recomputePersistedCaseStatus as d, cacheOperationTypeSchema as dt, setScopeCacheContext as en, evalChartBuiltinMetricSchema as et, runTouchesEval as f, cacheRecordingOpSchema as ft, setLatestRunInfoMap as g, spanCacheOptionsSchema as gt, getTargetEvalIds as h, serializedCacheSpanSchema as ht, getLatestRunInfos as i, evalChartTooltipExtraSchema as it, deriveStatusFromCaseRows as j, columnKindSchema as jt, getEvalDisplayStatus as k, columnDefSchema as kt, persistRunState as l, cacheListItemSchema as lt, buildEvalSummary as m, cacheStatusSchema as mt, generateRunId as n, defineEval as nn, evalChartConfigSchema as nt, loadPersistedRunSnapshots as o, evalChartsConfigSchema as ot, resolveArtifactPath as p, cacheRecordingSchema as pt, evalStatAggregateSchema as q, getCurrentScope as qt, getLastRunStatuses as r, getEvalRegistry as rn, evalChartMetricSchema as rt, nextShortIdFromSnapshots as s, cacheEntrySchema as st, executeRun as t, repoFile as tn, evalChartColorSchema as tt, recomputeEvalStatusesInRuns as u, cacheModeSchema as ut, loadConfig as v, traceAttributeDisplayFormatSchema as vt, sseEnvelopeSchema as w, traceSpanErrorSchema as wt, createFsCacheStore as x, traceAttributeDisplaySchema as xt, buildDeclaredColumnDefs as y, traceAttributeDisplayInputSchema as yt, llmCallMetricSchema as z, captureEvalSpanError as zt };
+export { evalChartAxisSchema as $, runInEvalScope as $t, deriveScopedSummaryFromCases as A, columnFormatSchema as At, llmCallsConfigSchema as B, evalSpan as Bt, updateManualScoreRequestSchema as C, traceDisplayInputConfigSchema as Ct, getNestedAttribute as D, traceSpanWarningSchema as Dt, extractLlmCalls as E, traceSpanSchema as Et, DEFAULT_LLM_CALLS_CONFIG as F, repoFileRefSchema as Ft, caseRowSchema as G, appendToEvalOutput as Gt, trialSelectionModeSchema as H, hashCacheKey as Ht, agentEvalsConfigSchema as I, runArtifactRefSchema as It, evalStatItemSchema as J, getEvalCaseInput as Jt, evalFreshnessStatusSchema as K, evalAssert as Kt, llmCallMetricFormatSchema as L, z$1 as Lt, deriveStatusFromChildStatuses as M, fileRefSchema as Mt, runManifestSchema as N, jsonCellSchema as Nt, getEvalTitle as O, cellValueSchema as Ot, runSummarySchema as P, numberDisplayOptionsSchema as Pt, evalChartAggregateSchema as Q, nextEvalId as Qt, llmCallMetricPlacementSchema as R, buildTraceTree as Rt, createRunRequestSchema as S, traceDisplayConfigSchema as St, extractCacheHits as T, traceSpanKindSchema as Tt, assertionFailureSchema as U, hashCacheKeySync as Ut, resolveLlmCallsConfig as V, evalTracer as Vt, caseDetailSchema as W, EvalAssertionError as Wt, evalSummarySchema as X, isInEvalScope as Xt, evalStatsConfigSchema as Y, incrementEvalOutput as Yt, scoreTraceSchema as Z, mergeEvalOutput as Zt, loadEvalModule as _, traceCacheRefSchema as _t, loadPersistedRunSnapshot as a, evalChartTypeSchema as at, normalizeScoreDef as b, traceAttributeDisplayPlacementSchema as bt, persistCaseDetail as c, cacheFileSchema as ct, recomputePersistedCaseStatus as d, cacheOperationTypeSchema as dt, setEvalOutput as en, evalChartBuiltinMetricSchema as et, runTouchesEval as f, cacheRecordingOpSchema as ft, setLatestRunInfoMap as g, spanCacheOptionsSchema as gt, getTargetEvalIds as h, serializedCacheSpanSchema as ht, getLatestRunInfos as i, getEvalRegistry as in, evalChartTooltipExtraSchema as it, deriveStatusFromCaseRows as j, columnKindSchema as jt, getEvalDisplayStatus as k, columnDefSchema as kt, persistRunState as l, cacheListItemSchema as lt, buildEvalSummary as m, cacheStatusSchema as mt, generateRunId as n, repoFile as nn, evalChartConfigSchema as nt, loadPersistedRunSnapshots as o, evalChartsConfigSchema as ot, resolveArtifactPath as p, cacheRecordingSchema as pt, evalStatAggregateSchema as q, getCurrentScope as qt, getLastRunStatuses as r, defineEval as rn, evalChartMetricSchema as rt, nextShortIdFromSnapshots as s, cacheEntrySchema as st, executeRun as t, setScopeCacheContext as tn, evalChartColorSchema as tt, recomputeEvalStatusesInRuns as u, cacheModeSchema as ut, loadConfig as v, traceAttributeDisplayFormatSchema as vt, sseEnvelopeSchema as w, traceSpanErrorSchema as wt, createFsCacheStore as x, traceAttributeDisplaySchema as xt, buildDeclaredColumnDefs as y, traceAttributeDisplayInputSchema as yt, llmCallMetricSchema as z, captureEvalSpanError as zt };

package/dist/{runner-DTP5Ui4_.mjs → runner-Gtlmvm3w.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-DrPk66xh.mjs";
-import "./src-CfprG1RW.mjs";
+import { n as createRunner } from "./cli-DLlRkyLH.mjs";
+import "./src-Bx-CV6Wo.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{runner-BzT3B9OF.mjs → runner-JrBz8ISs.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-DTP5Ui4_.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-Gtlmvm3w.mjs";
 export { getRunnerInstance, initRunner };

package/dist/src-Bx-CV6Wo.mjs

@@ -0,0 +1,3 @@
+import "./runOrchestration-DwqX9_T7.mjs";
+import "./cli-DLlRkyLH.mjs";
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -58,8 +58,8 @@
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/shared": "0.0.1",
-    "@agent-evals/sdk": "0.0.1"
+    "@agent-evals/sdk": "0.0.1",
+    "@agent-evals/shared": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && tsdown",

package/dist/src-CfprG1RW.mjs

@@ -1,3 +0,0 @@
-import "./runOrchestration-DA4Rh5g0.mjs";
-import "./cli-DrPk66xh.mjs";
-export {};