@ls-stack/agent-eval 0.61.0 → 0.61.2

package/dist/index.d.mts

@@ -2558,8 +2558,8 @@ declare const evalSummarySchema$1: z.ZodObject<{
       }>;
       label: z.ZodOptional<z.ZodString>;
       color: z.ZodOptional<z.ZodEnum<{
-        error: "error";
         success: "success";
+        error: "error";
         warning: "warning";
         accent: "accent";
         accentDim: "accentDim";
@@ -2582,8 +2582,8 @@ declare const evalSummarySchema$1: z.ZodObject<{
       }>;
       label: z.ZodOptional<z.ZodString>;
       color: z.ZodOptional<z.ZodEnum<{
-        error: "error";
         success: "success";
+        error: "error";
         warning: "warning";
         accent: "accent";
         accentDim: "accentDim";
@@ -2920,10 +2920,10 @@ declare const scoreTraceSchema: z.ZodObject<{
     namespace: z.ZodString;
     key: z.ZodString;
     status: z.ZodEnum<{
-      bypass: "bypass";
-      refresh: "refresh";
       hit: "hit";
       miss: "miss";
+      refresh: "refresh";
+      bypass: "bypass";
     }>;
     read: z.ZodOptional<z.ZodBoolean>;
     stored: z.ZodOptional<z.ZodBoolean>;
@@ -3092,10 +3092,10 @@ declare const caseDetailSchema$1: z.ZodObject<{
       namespace: z.ZodString;
       key: z.ZodString;
       status: z.ZodEnum<{
-        bypass: "bypass";
-        refresh: "refresh";
         hit: "hit";
         miss: "miss";
+        refresh: "refresh";
+        bypass: "bypass";
       }>;
       read: z.ZodOptional<z.ZodBoolean>;
       stored: z.ZodOptional<z.ZodBoolean>;
@@ -3213,10 +3213,10 @@ declare const caseDetailSchema$1: z.ZodObject<{
     namespace: z.ZodString;
     key: z.ZodString;
     status: z.ZodEnum<{
-      bypass: "bypass";
-      refresh: "refresh";
       hit: "hit";
       miss: "miss";
+      refresh: "refresh";
+      bypass: "bypass";
     }>;
     read: z.ZodOptional<z.ZodBoolean>;
     stored: z.ZodOptional<z.ZodBoolean>;
@@ -3283,8 +3283,8 @@ type EvalChartAggregate = z.infer<typeof evalChartAggregateSchema>;
  * not emit raw hex so authored evals stay decoupled from the web theme.
  */
 declare const evalChartColorSchema: z.ZodEnum<{
-  error: "error";
   success: "success";
+  error: "error";
   warning: "warning";
   accent: "accent";
   accentDim: "accentDim";
@@ -3312,8 +3312,8 @@ declare const evalChartMetricSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }>;
   label: z.ZodOptional<z.ZodString>;
   color: z.ZodOptional<z.ZodEnum<{
-    error: "error";
     success: "success";
+    error: "error";
     warning: "warning";
     accent: "accent";
     accentDim: "accentDim";
@@ -3336,8 +3336,8 @@ declare const evalChartMetricSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }>;
   label: z.ZodOptional<z.ZodString>;
   color: z.ZodOptional<z.ZodEnum<{
-    error: "error";
     success: "success";
+    error: "error";
     warning: "warning";
     accent: "accent";
     accentDim: "accentDim";
@@ -3395,8 +3395,8 @@ declare const evalChartConfigSchema: z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
-      error: "error";
       success: "success";
+      error: "error";
       warning: "warning";
       accent: "accent";
       accentDim: "accentDim";
@@ -3419,8 +3419,8 @@ declare const evalChartConfigSchema: z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
-      error: "error";
       success: "success";
+      error: "error";
       warning: "warning";
       accent: "accent";
       accentDim: "accentDim";
@@ -3485,8 +3485,8 @@ declare const evalChartsConfigSchema: z.ZodArray<z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
-      error: "error";
       success: "success";
+      error: "error";
       warning: "warning";
       accent: "accent";
       accentDim: "accentDim";
@@ -3509,8 +3509,8 @@ declare const evalChartsConfigSchema: z.ZodArray<z.ZodObject<{
     }>;
     label: z.ZodOptional<z.ZodString>;
     color: z.ZodOptional<z.ZodEnum<{
-      error: "error";
       success: "success";
+      error: "error";
       warning: "warning";
       accent: "accent";
       accentDim: "accentDim";
@@ -3589,9 +3589,9 @@ declare const runManifestSchema$1: z.ZodObject<{
     median: "median";
   }>>>;
   cacheMode: z.ZodOptional<z.ZodEnum<{
-    use: "use";
-    bypass: "bypass";
     refresh: "refresh";
+    bypass: "bypass";
+    use: "use";
   }>>;
 }, z.core.$strip>;
 /** Persisted lifecycle metadata for a single eval run. */
@@ -4575,9 +4575,9 @@ declare function extractApiCalls(spans: EvalTraceSpan$1[], config: ResolvedApiCa
  * - `refresh`: never read, always write (forces re-execution and overwrites).
  */
 declare const cacheModeSchema: z.ZodEnum<{
-  use: "use";
-  bypass: "bypass";
   refresh: "refresh";
+  bypass: "bypass";
+  use: "use";
 }>;
 /** Mode controlling how cached spans behave during a run. */
 type CacheMode = z.infer<typeof cacheModeSchema>;
@@ -4591,17 +4591,17 @@ declare const spanCacheOptionsSchema: z.ZodObject<{
 type SpanCacheOptions = z.infer<typeof spanCacheOptionsSchema>;
 /** Category of operation stored in the eval cache. */
 declare const cacheOperationTypeSchema: z.ZodEnum<{
-  span: "span";
   value: "value";
+  span: "span";
 }>;
 /** Category of operation stored in the eval cache. */
 type CacheOperationType = z.infer<typeof cacheOperationTypeSchema>;
 /** Status of a cache lookup recorded on a span or case scope. */
 declare const cacheStatusSchema: z.ZodEnum<{
-  bypass: "bypass";
-  refresh: "refresh";
   hit: "hit";
   miss: "miss";
+  refresh: "refresh";
+  bypass: "bypass";
 }>;
 /** Status of a cache lookup recorded on a span or case scope. */
 type CacheStatus = z.infer<typeof cacheStatusSchema>;
@@ -4618,10 +4618,10 @@ declare const traceCacheRefSchema: z.ZodObject<{
   namespace: z.ZodString;
   key: z.ZodString;
   status: z.ZodEnum<{
-    bypass: "bypass";
-    refresh: "refresh";
     hit: "hit";
     miss: "miss";
+    refresh: "refresh";
+    bypass: "bypass";
   }>;
   read: z.ZodOptional<z.ZodBoolean>;
   stored: z.ZodOptional<z.ZodBoolean>;
@@ -4817,8 +4817,8 @@ declare const cacheEntrySchema: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodOptional<z.ZodEnum<{
-    span: "span";
     value: "value";
+    span: "span";
   }>>;
   operationName: z.ZodOptional<z.ZodString>;
   spanName: z.ZodOptional<z.ZodString>;
@@ -4924,8 +4924,8 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodEnum<{
-    span: "span";
     value: "value";
+    span: "span";
   }>;
   operationName: z.ZodString;
   storedAt: z.ZodString;
@@ -4935,8 +4935,8 @@ declare const cacheDebugKeyEntrySchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodOptional<z.ZodEnum<{
-      span: "span";
       value: "value";
+      span: "span";
     }>>;
     operationName: z.ZodOptional<z.ZodString>;
     spanName: z.ZodOptional<z.ZodString>;
@@ -5042,8 +5042,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
   key: z.ZodString;
   namespace: z.ZodString;
   operationType: z.ZodOptional<z.ZodEnum<{
-    span: "span";
     value: "value";
+    span: "span";
   }>>;
   operationName: z.ZodOptional<z.ZodString>;
   spanName: z.ZodOptional<z.ZodString>;
@@ -5140,8 +5140,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodEnum<{
-      span: "span";
       value: "value";
+      span: "span";
     }>;
     operationName: z.ZodString;
     storedAt: z.ZodString;
@@ -5151,8 +5151,8 @@ declare const cacheEntryWithDebugKeySchema$1: z.ZodObject<{
       key: z.ZodString;
       namespace: z.ZodString;
       operationType: z.ZodOptional<z.ZodEnum<{
-        span: "span";
         value: "value";
+        span: "span";
       }>>;
       operationName: z.ZodOptional<z.ZodString>;
       spanName: z.ZodOptional<z.ZodString>;
@@ -5258,8 +5258,8 @@ declare const cacheFileSchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodOptional<z.ZodEnum<{
-      span: "span";
       value: "value";
+      span: "span";
     }>>;
     operationName: z.ZodOptional<z.ZodString>;
     spanName: z.ZodOptional<z.ZodString>;
@@ -5364,8 +5364,8 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
     key: z.ZodString;
     namespace: z.ZodString;
     operationType: z.ZodEnum<{
-      span: "span";
       value: "value";
+      span: "span";
     }>;
     operationName: z.ZodString;
     storedAt: z.ZodString;
@@ -5375,8 +5375,8 @@ declare const cacheDebugKeyFileSchema: z.ZodObject<{
       key: z.ZodString;
       namespace: z.ZodString;
       operationType: z.ZodOptional<z.ZodEnum<{
-        span: "span";
         value: "value";
+        span: "span";
       }>>;
       operationName: z.ZodOptional<z.ZodString>;
       spanName: z.ZodOptional<z.ZodString>;
@@ -5587,9 +5587,9 @@ declare const createRunRequestSchema$1: z.ZodObject<{
   temporary: z.ZodOptional<z.ZodBoolean>;
   cache: z.ZodOptional<z.ZodObject<{
     mode: z.ZodDefault<z.ZodEnum<{
-      use: "use";
-      bypass: "bypass";
       refresh: "refresh";
+      bypass: "bypass";
+      use: "use";
     }>>;
   }, z.core.$strip>>;
   manualInputs: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.61.0",
+  "version": "0.61.2",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -33,8 +33,8 @@
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/shared": "0.0.1",
-    "@agent-evals/sdk": "0.0.1"
+    "@agent-evals/sdk": "0.0.1",
+    "@agent-evals/shared": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",

package/skills/agent-eval/SKILL.md

@@ -212,11 +212,11 @@ See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape (forma
 - `tracingAssertions` is a single function that can be authored globally or locally on one eval when a finished-trace invariant should pass or fail the case without creating a fake score column. It receives the same `{ trace, input, case }` context as `deriveFromTracing`; call `evalAssert(...)` or `evalExpect(...)` inside it. Useful trace helpers include `trace.findSpan(name)`, `trace.findSpans(name)`, `trace.hasSpan(name)`, `trace.findSpansByKind(kind)`, `trace.findToolCallSpans()`, `trace.listToolCallSpanNames()`, `trace.hasToolCallSpan(name)`, `trace.getToolCallSpans(name)`, `trace.getToolCallSpanCount(toolName)`, `trace.hasToolCallSpanCount(toolName, expectedCalls)`, `trace.listSpanNames(kind?)`, `trace.listSpanNamesDfs(kind?)`, and `trace.flattenDfs()`. The tool-call helpers include both `kind: 'tool'` spans and imported execution spans recorded as `kind: 'tool_call'`. Tool-name checks and counts match the span `name` as well as GenAI/Mastra identity attributes such as `genAI["gen_ai.tool.name"]` and `mastra.entityName`; list helpers prefer those tool identity attributes when present. `getToolCallSpans(name)` returns one normalized object per matching call, including parsed `arguments`, parsed `result`, `description`, `toolType`, `attributes`, and the original `span`.
 - `traceDisplay` promotes selected span attributes into the trace tree and detail pane; it supports aggregation across subtrees (`scope`, `mode`) and user-defined `transform(...)` for derived views (e.g. currency conversion). See the `TraceDisplayInputConfig` type.
 - `llmCalls` (in `agent-evals.config.ts`) configures how LLM-call spans are summarized for review. Defaults to `kind: 'llm'` spans with `model`, `usage.*`, `latencyMs`, `input`, `output`, etc. read from conventional attribute paths. The default `steps` path reads an array from `span.attributes.steps`; if it is missing, direct child `model_step` spans are shown as that call's steps. Tool calls are aggregated from the configured `toolCalls` path plus step-level `toolCalls` on authored step arrays or direct `model_step` child spans, including Mastra's serialized `mastra.model_step.output` format, and child `tool_call` execution spans under each model step. `latencyMs` is time to first token; duration, total tokens, output tokens/sec, and USD costs are derived. Override `kinds` to broaden the filter, override `attributes.<field>` for non-default primitive span shapes, configure model-keyed `pricing` to derive USD costs from token counts, with nested `providers` entries for provider-specific rates, add `costCurrencies` to show converted cost columns in the expanded breakdown table only, add `derivedAttributes` to persist computed values back onto matching LLM spans before trace consumers run, and add entries to `metrics` to surface arbitrary user metrics (`format: 'string' | 'number' | 'duration' | 'json' | 'boolean'`, `placements: ['header' | 'body']`). `derivedAttributes` can be a keyed map for one-off fields or one callback that returns multiple path/value pairs. Derived keys are dot-paths under `span.attributes`; return `undefined` to skip one span or one returned key.
-- Default usage config derives missing eval outputs from matching LLM/API spans before `outputsSchema` and scores run: `apiCalls`, `costUsd`, `llmTurns`, `inputTokens`, `outputTokens`, `totalTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `reasoningTokens`, and `llmDurationMs`. Authored outputs and column overrides win. Default usage columns, stats, and charts use `hideIfNoValue: true`. Default LLM usage charts configure cost, input tokens, and output tokens separately and use `dedupeConsecutiveValues: true` to skip repeated adjacent chart values. `totalTokens` is input + output only; cache read/write tokens stay separate and affect `costUsd` at their own rates. `llmTurns` is the maximum per-call turn count in the case run, using configured steps when available and otherwise one turn per matched LLM call span. Derived base input cost uses `inputTokens - cachedInputTokens - cacheCreationInputTokens` so cache details are not double-counted. `cacheCreationInputTokens` is the total cache-write count; optional `cacheCreationInput1hTokens` only splits that total for 1-hour write pricing via `cacheCreationInput1hUsdPerMillion`. `llmDurationMs` sums elapsed matched LLM span durations; it is not time-to-first-token latency. Remove defaults globally or per eval with `removeDefaultConfig: true` or a key list such as `removeDefaultConfig: ['apiCalls', 'reasoningTokens']`.
+- Default usage config derives missing eval outputs from matching LLM/API spans before `outputsSchema` and scores run: `apiCalls`, `costUsd`, `llmTurns`, `inputTokens`, `outputTokens`, `totalTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `reasoningTokens`, and `llmDurationMs`. Authored outputs and column overrides win. The web UI fills in baseline run-health stats (`cases`, `passRate`, `duration`) and a pass-rate/duration history chart when an eval has not already authored equivalent run-health UI. If discovery metadata is missing but saved runs contain runtime columns such as `costUsd`, `inputTokens`, or `apiCalls`, the single-eval page can infer the standard usage stats and charts from those saved run values. Default usage columns, stats, and charts use `hideIfNoValue: true`. Default LLM usage charts configure cost, input tokens, and output tokens separately and use `dedupeConsecutiveValues: true` to skip repeated adjacent chart values. `totalTokens` is input + output only; cache read/write tokens stay separate and affect `costUsd` at their own rates. `llmTurns` is the maximum per-call turn count in the case run, using configured steps when available and otherwise one turn per matched LLM call span. Derived base input cost uses `inputTokens - cachedInputTokens - cacheCreationInputTokens` so cache details are not double-counted. `cacheCreationInputTokens` is the total cache-write count; optional `cacheCreationInput1hTokens` only splits that total for 1-hour write pricing via `cacheCreationInput1hUsdPerMillion`. `llmDurationMs` sums elapsed matched LLM span durations; it is not time-to-first-token latency. Remove defaults globally or per eval with `removeDefaultConfig: true` or a key list such as `removeDefaultConfig: ['apiCalls', 'reasoningTokens']`.
 - `apiCalls` (in `agent-evals.config.ts`) configures how API-call spans are summarized for review. Defaults to `kind: 'api'`, `'http'`, `'http.client'`, and `'fetch'` spans with `method`, `url`, `statusCode`, `request`, `routeAlias`, `response`, `requestBody`, `responseBody`, `headers`, `durationMs`, and `error` read from conventional attribute paths. Override `kinds` or `attributes.<field>` for external tracers. Set a per-span `routeAlias` attribute such as `/v3/tabs/:id` to group dynamic URL paths in API-call route labels and endpoint charts while preserving original URLs in row details. Add `derivedAttributes` as a keyed map or object-returning callback for computed persisted API span attributes, and add `metrics` with the same formats and placements as LLM-call metrics.
 - `runLogs` (in `agent-evals.config.ts`) controls case log capture. Use `runLogs: { captureConsole: false }` to keep console output in the terminal without persisting console calls to case details. Manual `evalLog(...)` calls are still captured. Captured log locations store the selected user-facing source frame and the full JavaScript stack so agents can inspect additional frames in persisted artifacts when diagnosing where a log came from.
 
-Stats rows and history charts can be authored via `stats` / `charts` on the eval definition. Global `stats` in `agent-evals.config.ts` combine with eval-level stats. Native stat kinds include `cases`, `passRate`, `duration`, and `cacheHits`; `cacheHits` shows Agent Eval operation-level cache hits over total cache operations (`hits/total`) from spans and `evalTracer.cache(...)` refs, not LLM provider prompt-cache read tokens such as `cachedInputTokens`. Cache-hit stats use a separate aggregate control and default to `sum`; `avg` is average per-case hit rate, and min/max/best/worst select cases by hit rate. `duration` aggregates per-case durations using the same modes as column stats. Usage stats and LLM usage charts are added by default unless removed with `removeDefaultConfig`. Column stats can override `format` and `numberFormat`, otherwise they inherit from the matching column. Duration and column stat aggregates support `avg`, `min`, `max`, `sum`, `best` (highest finite value), and `worst` (lowest finite value). Use `defaultStatAggregate` in `agent-evals.config.ts` to set the workspace-wide initial duration/column stat mode, or on an eval definition to override it for that eval. Number formats use `maxDecimalPlaces` to cap decimals and `minDecimalPlaces` to pad trailing zeroes. Without `maxDecimalPlaces`, the default cap is 3 decimal places. Stats and charts support `hideIfNoValue: true`. Charts support `dedupeConsecutiveValues: true` to omit consecutive points whose plotted metrics and tooltip extras match the previous kept point. Their shapes live in the types; no need to memorize the option set.
+Stats rows and history charts can be authored via `stats` / `charts` on the eval definition. Global `stats` in `agent-evals.config.ts` combine with eval-level stats. The web UI automatically supplies missing `cases`, `passRate`, and `duration` stats plus a pass-rate/duration history chart, including for a single completed run. Native stat kinds include `cases`, `passRate`, `duration`, and `cacheHits`; `cacheHits` shows Agent Eval operation-level cache hits over total cache operations (`hits/total`) from spans and `evalTracer.cache(...)` refs, not LLM provider prompt-cache read tokens such as `cachedInputTokens`. Cache-hit stats use a separate aggregate control and default to `sum`; `avg` is average per-case hit rate, and min/max/best/worst select cases by hit rate. `duration` aggregates per-case durations using the same modes as column stats. Usage stats and LLM usage charts are added by default unless removed with `removeDefaultConfig`. Column stats can override `format` and `numberFormat`, otherwise they inherit from the matching column. Duration and column stat aggregates support `avg`, `min`, `max`, `sum`, `best` (highest finite value), and `worst` (lowest finite value). Use `defaultStatAggregate` in `agent-evals.config.ts` to set the workspace-wide initial duration/column stat mode, or on an eval definition to override it for that eval. Number formats use `maxDecimalPlaces` to cap decimals and `minDecimalPlaces` to pad trailing zeroes. Without `maxDecimalPlaces`, the default cap is 3 decimal places. Stats and charts support `hideIfNoValue: true`. Charts support `dedupeConsecutiveValues: true` to omit consecutive points whose plotted metrics and tooltip extras match the previous kept point. Rendered charts with no plottable values show an unavailable state instead of a blank frame. Their shapes live in the types; no need to memorize the option set.
 
 ## Cached operations