@ls-stack/agent-eval 0.23.0 → 0.25.0

package/dist/{runner-B-SYzW8w.mjs → runner-B2f2TEjp.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-4yNYRvmF.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-cj1TkR-H.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-4yNYRvmF.mjs → runner-cj1TkR-H.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-D3QNOcPN.mjs";
-import "./src-CcXfWT4M.mjs";
+import { n as createRunner } from "./cli-ETfZ15RB.mjs";
+import "./src-CyNb2ycA.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/src-CyNb2ycA.mjs

@@ -0,0 +1,3 @@
+import "./runOrchestration-B31SV_Bq.mjs";
+import "./cli-ETfZ15RB.mjs";
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.23.0",
+  "version": "0.25.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"

package/skills/agent-eval/SKILL.md

@@ -274,10 +274,12 @@ See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape
   attribute paths. `latencyMs` is time to first token; duration, total tokens,
   tokens/sec, and USD costs are derived. Override `kinds` to broaden the filter,
   override `attributes.<field>` for non-default primitive span shapes, configure
-  `pricing` to derive USD costs from token counts by model/provider, and add
-  entries to `metrics` to surface arbitrary user metrics (`format: 'string' |
-'number' | 'duration' | 'json' | 'boolean'`, `placements: ['header' |
-'body']`).
+  `pricing` to derive USD costs from token counts by model/provider, add
+  `derivedAttributes` to persist computed values back onto matching LLM spans
+  before trace consumers run, and add entries to `metrics` to surface arbitrary
+  user metrics (`format: 'string' | 'number' | 'duration' | 'json' |
+'boolean'`, `placements: ['header' | 'body']`). `derivedAttributes` keys are
+  dot-paths under `span.attributes`; return `undefined` to skip one span.
 - Default usage config derives missing eval outputs from matching LLM/API spans
   before `outputsSchema` and scores run: `apiCalls`, `costUsd`, `llmTurns`,
   `inputTokens`, `outputTokens`, `totalTokens`, `cachedInputTokens`,
@@ -298,7 +300,8 @@ cacheCreationInputTokens` so cache details are not double-counted.
   and `'fetch'` spans with `method`, `url`, `statusCode`, `request`,
   `response`, `requestBody`, `responseBody`, `headers`, `durationMs`, and
   `error` read from conventional attribute paths. Override `kinds` or
-  `attributes.<field>` for external tracers, and add `metrics` with the same
+  `attributes.<field>` for external tracers, add `derivedAttributes` for
+  computed persisted API span attributes, and add `metrics` with the same
   formats and placements as LLM-call metrics.
 - `runLogs` (in `agent-evals.config.ts`) controls case log capture. Use
   `runLogs: { captureConsole: false }` to keep console output in the terminal
@@ -371,6 +374,10 @@ Mental model:
   across operations/evals, but the source-file fingerprint still participates
   in the final key. Shared namespaces are reusable across evals in the same
   file; evals in different files miss even with the same namespace and key.
+- Authored eval ids are unique within one eval file. The exact eval identity is
+  the workspace-relative file path plus eval id, so the same id can be reused in
+  different files. Case ids must be unique within one eval; duplicate case ids
+  are reported as run errors.
 - Cache keys should be deterministic primitives, arrays, and plain objects.
   `Buffer`, `ArrayBuffer`, and typed arrays hash by bytes. Native `Blob`/`File`
   keys use stable metadata by default (`type`, `size`, plus
@@ -459,7 +466,8 @@ When adding or changing evals:
    `evalTracer.cache(...)`; never cache operations whose external side effects
    you depend on.
 7. Sanity-check after changes: `agent-evals list`, then
-   `agent-evals run --eval <id>`.
+   `agent-evals run --eval <id>`; use `--file <path|glob>` to target one file
+   when multiple files use the same eval id.
 8. Locate saved artifacts with `agent-evals show-runs latest --json`, then read
    the relevant `summary.json`, `cases.jsonl`, `case-details/<case-id>.json`,
    or `traces/<case-id>.json` file directly.