@ls-stack/agent-eval 0.23.0 → 0.25.0

package/dist/index.d.mts

@@ -457,6 +457,7 @@ declare const evalStatsConfigSchema: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1
 type EvalStatsConfig = z$1.infer<typeof evalStatsConfigSchema>;
 /** Schema summarizing a discovered eval for list and overview screens. */
 declare const evalSummarySchema: z$1.ZodObject<{
+  key: z$1.ZodDefault<z$1.ZodString>;
   id: z$1.ZodString;
   title: z$1.ZodOptional<z$1.ZodString>;
   filePath: z$1.ZodString;
@@ -635,6 +636,8 @@ declare const evalSummarySchema: z$1.ZodObject<{
 type EvalSummary = z$1.infer<typeof evalSummarySchema>;
 /** Schema for one case row in an eval run result table. */
 declare const caseRowSchema: z$1.ZodObject<{
+  evalKey: z$1.ZodOptional<z$1.ZodString>;
+  caseKey: z$1.ZodOptional<z$1.ZodString>;
   caseId: z$1.ZodString;
   evalId: z$1.ZodString;
   status: z$1.ZodEnum<{
@@ -796,6 +799,8 @@ declare const scoreTraceSchema: z$1.ZodObject<{
 type ScoreTrace = z$1.infer<typeof scoreTraceSchema>;
 /** Schema for the detailed payload shown when opening a specific case. */
 declare const caseDetailSchema: z$1.ZodObject<{
+  evalKey: z$1.ZodOptional<z$1.ZodString>;
+  caseKey: z$1.ZodOptional<z$1.ZodString>;
   caseId: z$1.ZodString;
   evalId: z$1.ZodString;
   status: z$1.ZodEnum<{
@@ -1009,6 +1014,43 @@ declare const caseDetailSchema: z$1.ZodObject<{
 }, z$1.core.$strip>;
 /** Full case payload including inputs, trace, outputs, and failures. */
 type CaseDetail = z$1.infer<typeof caseDetailSchema>;
+/** Schema for discovery problems that should be shown before running evals. */
+declare const discoveryIssueSchema: z$1.ZodObject<{
+  type: z$1.ZodEnum<{
+    "duplicate-eval-id": "duplicate-eval-id";
+  }>;
+  severity: z$1.ZodEnum<{
+    error: "error";
+  }>;
+  filePath: z$1.ZodString;
+  evalId: z$1.ZodString;
+  message: z$1.ZodString;
+}, z$1.core.$strip>;
+/** Discovery problem found while scanning eval files. */
+type DiscoveryIssue = z$1.infer<typeof discoveryIssueSchema>;
+//#endregion
+//#region ../shared/src/evalIdentity.d.ts
+/** Build the stable identity for one eval inside a workspace. */
+declare function buildEvalKey(params: {
+  filePath: string;
+  evalId: string;
+}): string;
+/** Build the stable identity for one eval case inside a workspace. */
+declare function buildCaseKey(params: {
+  filePath: string;
+  evalId: string;
+  caseId: string;
+}): string;
+/** Return the collision-safe eval key stored on a row, falling back for legacy data. */
+declare function getCaseRowEvalKey(row: {
+  evalKey?: string;
+  evalId: string;
+}): string;
+/** Return the collision-safe case key stored on a row, falling back for legacy data. */
+declare function getCaseRowCaseKey(row: {
+  caseKey?: string;
+  caseId: string;
+}): string;
 //#endregion
 //#region ../shared/src/schemas/chart.d.ts
 /** Chart type rendered for a single eval history chart. */
@@ -1338,6 +1380,8 @@ declare const runManifestSchema: z$1.ZodObject<{
       evalIds: "evalIds";
       caseIds: "caseIds";
     }>;
+    evalKeys: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
+    files: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     evalIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   }, z$1.core.$strip>;
@@ -1539,13 +1583,35 @@ declare const apiCallMetricPlacementSchema: z$1.ZodEnum<{
 }>;
 /** Placement option for an API-call metric. */
 type ApiCallMetricPlacement = z$1.infer<typeof apiCallMetricPlacementSchema>;
+/** Context passed to an LLM/API-call derived attribute function. */
+type CallDerivedAttributeContext = {
+  /** Raw attributes from the matching trace span. */attributes: Record<string, unknown> | undefined; /** Matching trace span. */
+  span: EvalTraceSpan; /** Dot-path helper for reading from `span.attributes`. */
+  get: (path: string) => unknown;
+};
+/**
+ * Runner-side function used to derive one new span attribute from a matching
+ * LLM/API-call span. Return `undefined` to omit the attribute for that span.
+ */
+type CallDerivedAttribute = (ctx: CallDerivedAttributeContext) => unknown;
+/** One resolved derived span attribute rule. */
+type ResolvedCallDerivedAttribute = {
+  /** Dot-path where the derived value is persisted on `span.attributes`. */path: string;
+  /**
+   * Function that derives the persisted value for each matching span. Omitted
+   * after this config is serialized to the browser.
+   */
+  compute?: CallDerivedAttribute;
+};
 /**
  * Schema for a single user-defined metric attached to LLM call rows.
  *
  * Each metric reads `path` from the span's `attributes` and renders the value
- * with the configured `format` and `numberFormat`. `placements` controls
- * whether the metric appears as a chip on the collapsed row header, as a row
- * inside the expanded body, or both. Defaults to `['body']` when omitted.
+ * with the configured `format` and `numberFormat`. Use
+ * `llmCalls.derivedAttributes` when a metric should read a value computed from
+ * other attributes. `placements` controls whether the metric appears as a chip
+ * on the collapsed row header, as a row inside the expanded body, or both.
+ * Defaults to `['body']` when omitted.
  */
 declare const llmCallMetricSchema: z$1.ZodObject<{
   label: z$1.ZodString;
@@ -1570,9 +1636,11 @@ type LlmCallMetric = z$1.infer<typeof llmCallMetricSchema>;
  * Schema for a single user-defined metric attached to API call rows.
  *
  * Each metric reads `path` from the span's `attributes` and renders the value
- * with the configured `format` and `numberFormat`. `placements` controls
- * whether the metric appears as a chip on the collapsed row header, as a row
- * inside the expanded body, or both. Defaults to `['body']` when omitted.
+ * with the configured `format` and `numberFormat`. Use
+ * `apiCalls.derivedAttributes` when a metric should read a value computed from
+ * other attributes. `placements` controls whether the metric appears as a chip
+ * on the collapsed row header, as a row inside the expanded body, or both.
+ * Defaults to `['body']` when omitted.
  */
 declare const apiCallMetricSchema: z$1.ZodObject<{
   label: z$1.ZodString;
@@ -1629,6 +1697,7 @@ declare const llmCallsConfigSchema: z$1.ZodObject<{
     reasoning: z$1.ZodOptional<z$1.ZodString>;
     toolCalls: z$1.ZodOptional<z$1.ZodString>;
   }, z$1.core.$strip>>;
+  derivedAttributes: z$1.ZodOptional<z$1.ZodRecord<z$1.ZodString, z$1.ZodCustom<CallDerivedAttribute, CallDerivedAttribute>>>;
   pricing: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
     model: z$1.ZodString;
     provider: z$1.ZodOptional<z$1.ZodString>;
@@ -1674,6 +1743,7 @@ declare const apiCallsConfigSchema: z$1.ZodObject<{
     durationMs: z$1.ZodOptional<z$1.ZodString>;
     error: z$1.ZodOptional<z$1.ZodString>;
   }, z$1.core.$strip>>;
+  derivedAttributes: z$1.ZodOptional<z$1.ZodRecord<z$1.ZodString, z$1.ZodCustom<CallDerivedAttribute, CallDerivedAttribute>>>;
   metrics: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
     label: z$1.ZodString;
     tooltip: z$1.ZodOptional<z$1.ZodString>;
@@ -1720,6 +1790,7 @@ type ResolvedLlmCallsConfig = {
     reasoning: string;
     toolCalls: string;
   };
+  derivedAttributes: ResolvedCallDerivedAttribute[];
   metrics: ResolvedLlmCallMetric[];
   pricing: ResolvedLlmCallPricing[];
 };
@@ -1738,6 +1809,7 @@ type ResolvedApiCallsConfig = {
     durationMs: string;
     error: string;
   };
+  derivedAttributes: ResolvedCallDerivedAttribute[];
   metrics: ResolvedApiCallMetric[];
 };
 /** Fully-resolved LLM-call metric used by the runner and UI. */
@@ -1983,6 +2055,7 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
       reasoning: z$1.ZodOptional<z$1.ZodString>;
       toolCalls: z$1.ZodOptional<z$1.ZodString>;
     }, z$1.core.$strip>>;
+    derivedAttributes: z$1.ZodOptional<z$1.ZodRecord<z$1.ZodString, z$1.ZodCustom<CallDerivedAttribute, CallDerivedAttribute>>>;
     pricing: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
       model: z$1.ZodString;
       provider: z$1.ZodOptional<z$1.ZodString>;
@@ -2037,6 +2110,7 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
       durationMs: z$1.ZodOptional<z$1.ZodString>;
       error: z$1.ZodOptional<z$1.ZodString>;
     }, z$1.core.$strip>>;
+    derivedAttributes: z$1.ZodOptional<z$1.ZodRecord<z$1.ZodString, z$1.ZodCustom<CallDerivedAttribute, CallDerivedAttribute>>>;
     metrics: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
       label: z$1.ZodString;
       tooltip: z$1.ZodOptional<z$1.ZodString>;
@@ -2067,6 +2141,20 @@ declare const agentEvalsConfigSchema: z$1.ZodObject<{
   }, z$1.core.$strip>>;
 }, z$1.core.$strip>;
 //#endregion
+//#region ../shared/src/utils/deriveCallAttributes.d.ts
+/**
+ * Persist configured derived attributes onto matching LLM/API spans.
+ *
+ * These derived attributes are applied before trace consumers run, so
+ * `deriveFromTracing`, default usage extraction, trace display, and call
+ * metrics can all read them by normal dot-path lookup.
+ */
+declare function applyDerivedCallAttributes(params: {
+  spans: EvalTraceSpan[];
+  llmCallsConfig: ResolvedLlmCallsConfig;
+  apiCallsConfig: ResolvedApiCallsConfig;
+}): EvalTraceSpan[];
+//#endregion
 //#region ../shared/src/utils/extractLlmCalls.d.ts
 /** Resolved value for one user-defined metric on an LLM call row. */
 type LlmCallMetricValue = {
@@ -2728,6 +2816,8 @@ declare const createRunRequestSchema: z$1.ZodObject<{
       evalIds: "evalIds";
       caseIds: "caseIds";
     }>;
+    evalKeys: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
+    files: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     evalIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
     caseIds: z$1.ZodOptional<z$1.ZodArray<z$1.ZodString>>;
   }, z$1.core.$strip>;
@@ -3520,7 +3610,8 @@ type CacheClearFilter = {
 type EvalRunner = {
   /** Load workspace config, discover evals, and start file watching when enabled. */init(): Promise<void>; /** Return the currently discovered eval summaries for the active workspace. */
   getEvals(): EvalSummary[]; /** Look up one discovered eval by id. */
-  getEval(id: string): EvalSummary | undefined; /** Re-scan configured eval files and emit a discovery update to listeners. */
+  getEval(id: string): EvalSummary | undefined; /** Return discovery errors that should be shown before running evals. */
+  getDiscoveryIssues(): DiscoveryIssue[]; /** Re-scan configured eval files and emit a discovery update to listeners. */
   refreshDiscovery(): Promise<void>;
   startRun(request: CreateRunRequest): Promise<{
     manifest: RunManifest;
@@ -3573,11 +3664,19 @@ type EvalRunner = {
    * Remove cache entries matching `filter`, or all entries when no filter is
    * supplied.
    */
-  clearCache(filter?: CacheClearFilter): Promise<void>; /** Recompute persisted case and run statuses for terminal runs touching one eval. */
-  recomputeStatusesForEval(evalId: string): Promise<{
+  clearCache(filter?: CacheClearFilter): Promise<void>;
+  /**
+   * Recompute persisted case and run statuses for terminal runs touching one
+   * eval. Accepts the exact eval key, with a legacy fallback for unique eval ids.
+   */
+  recomputeStatusesForEval(evalKey: string): Promise<{
     updatedRuns: number;
-  }>; /** Delete terminal persisted runs that touch one eval from in-memory history and disk. */
-  cleanRunsForEval(evalId: string): Promise<{
+  }>;
+  /**
+   * Delete terminal persisted runs that touch one eval from memory and disk.
+   * Accepts the exact eval key, with a legacy fallback for unique eval ids.
+   */
+  cleanRunsForEval(evalKey: string): Promise<{
     deletedRuns: number;
   }>; /** Persist a UI-authored manual score for one case and recompute affected summaries. */
   updateManualScore(params: {
@@ -3623,4 +3722,4 @@ declare function createRunner({
  */
 declare function runCli(argv: string[]): Promise<void>;
 //#endregion
-export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheActivityEntry, type CacheAdapter, type CacheDebugKeyEntry, type CacheDebugKeyFile, type CacheDebugKeyWrite, type CacheEntry, type CacheEntryWithDebugKey, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DefaultConfigKey, type DerivedStatus, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalRuntimeScope, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStartTime, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTimeUnit, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallPricing, type LlmCallsConfigInput, type NumberDisplayOptions, type RemoveDefaultConfig, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedLlmCallMetric, type ResolvedLlmCallPricing, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunLogEntry, type RunLogLevel, type RunLogLocation, type RunLogPhase, type RunLogsConfigInput, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SerializedCacheValue, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, advanceEvalTime, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheDebugKeyEntrySchema, cacheDebugKeyFileSchema, cacheEntrySchema, cacheEntryWithDebugKeySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defaultConfigKeySchema, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalLog, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalStartTime, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallPricingSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, removeDefaultConfigSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runLogEntrySchema, runLogLevelSchema, runLogLocationSchema, runLogPhaseSchema, runLogsConfigSchema, runManifestSchema, runSummarySchema, scoreTraceSchema, serializeCacheRecording, serializeCacheValue, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+export { type AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheActivityEntry, type CacheAdapter, type CacheDebugKeyEntry, type CacheDebugKeyFile, type CacheDebugKeyWrite, type CacheEntry, type CacheEntryWithDebugKey, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheStatus, type CallDerivedAttribute, type CallDerivedAttributeContext, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type CreateRunRequest, DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, type DefaultConfigKey, type DerivedStatus, type DiscoveryIssue, EvalAssertionError, type EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, type EvalDefinition, type EvalDeriveContext, type EvalDisplayStatus, type EvalExecuteContext, type EvalFreshnessStatus, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalRuntimeScope, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStartTime, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, type EvalTimeUnit, type EvalTraceSpan, type EvalTraceSpanError, type EvalTraceSpanWarning, type EvalTraceTree, type FileRef, type JsonCell, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallPricing, type LlmCallsConfigInput, type NumberDisplayOptions, type RemoveDefaultConfig, type RepoFileRef, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedCallDerivedAttribute, type ResolvedLlmCallMetric, type ResolvedLlmCallPricing, type ResolvedLlmCallsConfig, type RunArtifactRef, type RunInEvalScopeOptions, type RunLogEntry, type RunLogLevel, type RunLogLocation, type RunLogPhase, type RunLogsConfigInput, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SerializedCacheValue, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, advanceEvalTime, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, applyDerivedCallAttributes, assertionFailureSchema, buildCaseKey, buildEvalKey, buildTraceTree, cacheDebugKeyEntrySchema, cacheDebugKeyFileSchema, cacheEntrySchema, cacheEntryWithDebugKeySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defaultConfigKeySchema, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, deserializeCacheRecording, deserializeCacheValue, discoveryIssueSchema, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalLog, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, fileRefSchema, getCaseRowCaseKey, getCaseRowEvalKey, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalStartTime, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallPricingSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, removeDefaultConfigSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runLogEntrySchema, runLogLevelSchema, runLogLocationSchema, runLogPhaseSchema, runLogsConfigSchema, runManifestSchema, runSummarySchema, scoreTraceSchema, serializeCacheRecording, serializeCacheValue, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as removeDefaultConfigSchema, $t as columnKindSchema, A as extractApiCalls, An as runInEvalScope, At as cacheFileSchema, B as DEFAULT_API_CALLS_CONFIG, Bt as traceAttributeDisplayFormatSchema, Cn as getEvalCaseInput, Ct as evalChartTooltipExtraSchema, D as sseEnvelopeSchema, Dn as mergeEvalOutput, Dt as cacheDebugKeyFileSchema, E as updateManualScoreRequestSchema, En as isInEvalScope, Et as cacheDebugKeyEntrySchema, F as deriveScopedSummaryFromCases, Fn as repoFile, Ft as cacheRecordingSchema, G as apiCallMetricSchema, Gt as traceDisplayInputConfigSchema, H as agentEvalsConfigSchema, Ht as traceAttributeDisplayPlacementSchema, I as deriveStatusFromCaseRows, In as defineEval, It as cacheStatusSchema, J as llmCallMetricFormatSchema, Jt as traceSpanSchema, K as apiCallsConfigSchema, Kt as traceSpanErrorSchema, L as deriveStatusFromChildStatuses, Ln as getEvalRegistry, Lt as serializedCacheSpanSchema, M as getNestedAttribute, Mn as setEvalOutput, Mt as cacheModeSchema, N as getEvalTitle, Nn as setScopeCacheContext, Nt as cacheOperationTypeSchema, O as extractCacheEntries, On as nextEvalId, Ot as cacheEntrySchema, P as getEvalDisplayStatus, Pn as startEvalBackgroundJob, Pt as cacheRecordingOpSchema, Q as llmCallsConfigSchema, Qt as columnFormatSchema, R as runManifestSchema, Rt as spanCacheOptionsSchema, Sn as getCurrentScope, St as evalChartMetricSchema, T as createRunRequestSchema, Tn as incrementEvalOutput, Tt as evalChartsConfigSchema, U as apiCallMetricFormatSchema, Ut as traceAttributeDisplaySchema, V as DEFAULT_LLM_CALLS_CONFIG, Vt as traceAttributeDisplayInputSchema, W as apiCallMetricPlacementSchema, Wt as traceDisplayConfigSchema, X as llmCallMetricSchema, Xt as cellValueSchema, Y as llmCallMetricPlacementSchema, Yt as traceSpanWarningSchema, Z as llmCallPricingSchema, Zt as columnDefSchema, _n as advanceEvalTime, _t as evalChartAggregateSchema, an as z, at as caseDetailSchema, bn as evalAssert, bt as evalChartColorSchema, cn as evalSpan, ct as evalStatAggregateSchema, dn as hashCacheKeySync, dt as evalSummarySchema, en as fileRefSchema, et as resolveApiCallsConfig, fn as deserializeCacheRecording, ft as runLogEntrySchema, gn as EvalAssertionError, gt as scoreTraceSchema, hn as serializeCacheValue, ht as runLogPhaseSchema, in as runArtifactRefSchema, it as assertionFailureSchema, j as extractLlmCalls, jn as runInExistingEvalScope, jt as cacheListItemSchema, k as extractCacheHits, kn as runInEvalRuntimeScope, kt as cacheEntryWithDebugKeySchema, ln as evalTracer, lt as evalStatItemSchema, mn as serializeCacheRecording, mt as runLogLocationSchema, nn as numberDisplayOptionsSchema, nt as runLogsConfigSchema, on as buildTraceTree, ot as caseRowSchema, pn as deserializeCacheValue, pt as runLogLevelSchema, q as defaultConfigKeySchema, qt as traceSpanKindSchema, rn as repoFileRefSchema, rt as trialSelectionModeSchema, sn as captureEvalSpanError, st as evalFreshnessStatusSchema, tn as jsonCellSchema, tt as resolveLlmCallsConfig, un as hashCacheKey, ut as evalStatsConfigSchema, vn as appendToEvalOutput, vt as evalChartAxisSchema, wn as getEvalStartTime, wt as evalChartTypeSchema, xn as evalLog, xt as evalChartConfigSchema, yt as evalChartBuiltinMetricSchema, z as runSummarySchema, zt as traceCacheRefSchema } from "./runOrchestration-CimthgI7.mjs";
-import { n as createRunner, t as runCli } from "./cli-D3QNOcPN.mjs";
-import "./src-CcXfWT4M.mjs";
-export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, advanceEvalTime, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheDebugKeyEntrySchema, cacheDebugKeyFileSchema, cacheEntrySchema, cacheEntryWithDebugKeySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defaultConfigKeySchema, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalLog, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalStartTime, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallPricingSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, removeDefaultConfigSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runLogEntrySchema, runLogLevelSchema, runLogLocationSchema, runLogPhaseSchema, runLogsConfigSchema, runManifestSchema, runSummarySchema, scoreTraceSchema, serializeCacheRecording, serializeCacheValue, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };
+import { $ as llmCallsConfigSchema, $t as traceSpanKindSchema, A as extractApiCalls, An as getEvalStartTime, At as evalChartTypeSchema, B as runSummarySchema, Bn as startEvalBackgroundJob, Bt as cacheRecordingOpSchema, Cn as advanceEvalTime, Ct as evalChartAggregateSchema, D as sseEnvelopeSchema, Dn as evalLog, Dt as evalChartConfigSchema, E as updateManualScoreRequestSchema, En as evalAssert, Et as evalChartColorSchema, F as getEvalDisplayStatus, Fn as runInEvalRuntimeScope, Ft as cacheEntryWithDebugKeySchema, G as apiCallMetricPlacementSchema, Gt as traceCacheRefSchema, H as DEFAULT_LLM_CALLS_CONFIG, Hn as defineEval, Ht as cacheStatusSchema, I as deriveScopedSummaryFromCases, In as runInEvalScope, It as cacheFileSchema, J as defaultConfigKeySchema, Jt as traceAttributeDisplayPlacementSchema, K as apiCallMetricSchema, Kt as traceAttributeDisplayFormatSchema, L as deriveStatusFromCaseRows, Ln as runInExistingEvalScope, Lt as cacheListItemSchema, M as applyDerivedCallAttributes, Mn as isInEvalScope, Mt as cacheDebugKeyEntrySchema, N as getNestedAttribute, Nn as mergeEvalOutput, Nt as cacheDebugKeyFileSchema, O as extractCacheEntries, On as getCurrentScope, Ot as evalChartMetricSchema, P as getEvalTitle, Pn as nextEvalId, Pt as cacheEntrySchema, Q as llmCallPricingSchema, Qt as traceSpanErrorSchema, R as deriveStatusFromChildStatuses, Rn as setEvalOutput, Rt as cacheModeSchema, Sn as EvalAssertionError, St as scoreTraceSchema, T as createRunRequestSchema, Tt as evalChartBuiltinMetricSchema, U as agentEvalsConfigSchema, Un as getEvalRegistry, Ut as serializedCacheSpanSchema, V as DEFAULT_API_CALLS_CONFIG, Vn as repoFile, Vt as cacheRecordingSchema, W as apiCallMetricFormatSchema, Wt as spanCacheOptionsSchema, X as llmCallMetricPlacementSchema, Xt as traceDisplayConfigSchema, Y as llmCallMetricFormatSchema, Yt as traceAttributeDisplaySchema, Z as llmCallMetricSchema, Zt as traceDisplayInputConfigSchema, _n as hashCacheKeySync, _t as evalSummarySchema, an as columnKindSchema, at as buildCaseKey, bn as serializeCacheRecording, bt as runLogLocationSchema, cn as numberDisplayOptionsSchema, ct as getCaseRowEvalKey, dn as z, dt as caseRowSchema, en as traceSpanSchema, et as removeDefaultConfigSchema, fn as buildTraceTree, ft as discoveryIssueSchema, gn as hashCacheKey, gt as evalStatsConfigSchema, hn as evalTracer, ht as evalStatItemSchema, in as columnFormatSchema, it as trialSelectionModeSchema, j as extractLlmCalls, jn as incrementEvalOutput, jt as evalChartsConfigSchema, k as extractCacheHits, kn as getEvalCaseInput, kt as evalChartTooltipExtraSchema, ln as repoFileRefSchema, lt as assertionFailureSchema, mn as evalSpan, mt as evalStatAggregateSchema, nn as cellValueSchema, nt as resolveLlmCallsConfig, on as fileRefSchema, ot as buildEvalKey, pn as captureEvalSpanError, pt as evalFreshnessStatusSchema, q as apiCallsConfigSchema, qt as traceAttributeDisplayInputSchema, rn as columnDefSchema, rt as runLogsConfigSchema, sn as jsonCellSchema, st as getCaseRowCaseKey, tn as traceSpanWarningSchema, tt as resolveApiCallsConfig, un as runArtifactRefSchema, ut as caseDetailSchema, vn as deserializeCacheRecording, vt as runLogEntrySchema, wn as appendToEvalOutput, wt as evalChartAxisSchema, xn as serializeCacheValue, xt as runLogPhaseSchema, yn as deserializeCacheValue, yt as runLogLevelSchema, z as runManifestSchema, zn as setScopeCacheContext, zt as cacheOperationTypeSchema } from "./runOrchestration-B31SV_Bq.mjs";
+import { n as createRunner, t as runCli } from "./cli-ETfZ15RB.mjs";
+import "./src-CyNb2ycA.mjs";
+export { DEFAULT_API_CALLS_CONFIG, DEFAULT_LLM_CALLS_CONFIG, EvalAssertionError, advanceEvalTime, agentEvalsConfigSchema, apiCallMetricFormatSchema, apiCallMetricPlacementSchema, apiCallMetricSchema, apiCallsConfigSchema, appendToEvalOutput, applyDerivedCallAttributes, assertionFailureSchema, buildCaseKey, buildEvalKey, buildTraceTree, cacheDebugKeyEntrySchema, cacheDebugKeyFileSchema, cacheEntrySchema, cacheEntryWithDebugKeySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, cacheStatusSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defaultConfigKeySchema, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, deserializeCacheRecording, deserializeCacheValue, discoveryIssueSchema, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalLog, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, fileRefSchema, getCaseRowCaseKey, getCaseRowEvalKey, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalStartTime, getEvalTitle, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, llmCallMetricFormatSchema, llmCallMetricPlacementSchema, llmCallMetricSchema, llmCallPricingSchema, llmCallsConfigSchema, mergeEvalOutput, nextEvalId, numberDisplayOptionsSchema, removeDefaultConfigSchema, repoFile, repoFileRefSchema, resolveApiCallsConfig, resolveLlmCallsConfig, runArtifactRefSchema, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, runLogEntrySchema, runLogLevelSchema, runLogLocationSchema, runLogPhaseSchema, runLogsConfigSchema, runManifestSchema, runSummarySchema, scoreTraceSchema, serializeCacheRecording, serializeCacheValue, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, startEvalBackgroundJob, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceCacheRefSchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.mjs

@@ -1,4 +1,4 @@
-import { R as runManifestSchema, T as createRunRequestSchema, Tt as evalChartsConfigSchema, Zt as columnDefSchema, b as loadConfig, t as executeRun, ut as evalStatsConfigSchema, v as parseEvalMetas, w as createFsCacheStore, yn as configureEvalRunLogs, z as runSummarySchema } from "./runOrchestration-CimthgI7.mjs";
+import { B as runSummarySchema, T as createRunRequestSchema, Tn as configureEvalRunLogs, b as loadConfig, gt as evalStatsConfigSchema, jt as evalChartsConfigSchema, ot as buildEvalKey, r as getTargetEvals$1, rn as columnDefSchema, t as executeRun, v as parseEvalDiscovery, w as createFsCacheStore, z as runManifestSchema } from "./runOrchestration-B31SV_Bq.mjs";
 import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";
@@ -6,6 +6,7 @@ import { z } from "zod/v4";
 import { glob } from "glob";
 //#region ../runner/src/runChild.ts
 const evalMetaSchema = z.object({
+	key: z.string(),
 	id: z.string(),
 	title: z.string().optional(),
 	filePath: z.string(),
@@ -36,8 +37,10 @@ function getConfiguredConcurrency(configConcurrency) {
 	return Math.max(1, Math.floor(configConcurrency));
 }
 function getTargetEvals(params) {
-	if (params.request.target.evalIds && params.request.target.evalIds.length > 0) return params.request.target.evalIds.map((id) => params.evals.get(id)).filter((entry) => entry !== void 0);
-	return [...params.evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+	return getTargetEvals$1({
+		evals: params.evals.values(),
+		request: params.request
+	});
 }
 function toWorkspaceRelativePath(params) {
 	return relative(params.workspaceRoot, params.filePath).replaceAll("\\", "/");
@@ -55,21 +58,29 @@ async function discoverRunEvals(params) {
 	for (const filePath of discovered) {
 		const source = await readFile(filePath, "utf-8");
 		const sourceFingerprint = getSourceFingerprint(source);
-		const metas = parseEvalMetas(filePath, source);
-		for (const meta of metas) evals.set(meta.id, {
-			id: meta.id,
-			title: meta.title,
-			filePath: toWorkspaceRelativePath({
+		const metas = parseEvalDiscovery(filePath, source).metas;
+		for (const meta of metas) {
+			const relativeFilePath = toWorkspaceRelativePath({
 				filePath: meta.filePath,
 				workspaceRoot: params.workspaceRoot
-			}),
-			sourceFilePath: meta.filePath,
-			sourceFingerprint,
-			columnDefs: [],
-			caseCount: null
-		});
+			});
+			const key = buildEvalKey({
+				filePath: relativeFilePath,
+				evalId: meta.id
+			});
+			evals.set(key, {
+				key,
+				id: meta.id,
+				title: meta.title,
+				filePath: relativeFilePath,
+				sourceFilePath: meta.filePath,
+				sourceFingerprint,
+				columnDefs: [],
+				caseCount: null
+			});
+		}
 	}
-	return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+	return [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath) || a.id.localeCompare(b.id));
 }
 async function readContext(contextPath) {
 	if (contextPath === void 0) throw new Error("Missing run child context path");
@@ -93,7 +104,7 @@ async function main() {
 		config,
 		workspaceRoot: context.workspaceRoot
 	});
-	const evals = new Map(evalMetas.map((evalMeta) => [evalMeta.id, evalMeta]));
+	const evals = new Map(evalMetas.map((evalMeta) => [evalMeta.key, evalMeta]));
 	const lastRunStatusMap = /* @__PURE__ */ new Map();
 	const latestRunInfoMap = /* @__PURE__ */ new Map();
 	await executeRun({
@@ -108,7 +119,6 @@ async function main() {
 		request: context.request,
 		runDir: context.runDir,
 		config,
-		evals,
 		cacheStore,
 		lastRunStatusMap,
 		latestRunInfoMap,
@@ -123,7 +133,7 @@ async function main() {
 		workspaceRoot: context.workspaceRoot,
 		getSourceFingerprint,
 		getConfiguredConcurrency: () => getConfiguredConcurrency(config.concurrency),
-		getSortedEvalMetas: () => [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath)),
+		getSortedEvalMetas: () => [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath) || a.id.localeCompare(b.id)),
 		getTargetEvals: (request) => getTargetEvals({
 			evals,
 			request