@axlsdk/eval 0.13.4 → 0.13.6

package/README.md

@@ -107,7 +107,7 @@ Scorers rate each output on a 0-1 scale. You can mix deterministic and LLM score
 
 ### Deterministic scorers
 
-Pure functions — fast, free, deterministic. The `score` callback receives `(output, input, annotations?)`:
+Pure functions — fast, free, deterministic. The `score` callback receives `(output, input, annotations?)` and returns a number (0-1) or a `ScorerResult` with metadata:
 
 ```typescript
 import { scorer } from '@axlsdk/eval';
@@ -118,6 +118,16 @@ const containsAnswer = scorer({
   score: (output, _input, annotations) =>
     String(output).includes(String(annotations?.answer)) ? 1 : 0,
 });
+
+// Returning rich metadata via ScorerResult
+const lengthScore = scorer({
+  name: 'length',
+  description: 'Rates output by character length',
+  score: (output) => {
+    const len = String(output).length;
+    return { score: Math.min(len / 500, 1), metadata: { charCount: len } };
+  },
+});
 ```
 
 ### LLM scorers
@@ -135,7 +145,7 @@ const qualityJudge = llmScorer({
 });
 ```
 
-The default schema is `{ score: number, reasoning: string }` — the LLM returns a 0-1 score with an explanation. For custom scoring dimensions, provide your own schema:
+The default schema is `z.object({ score: z.number().min(0).max(1), reasoning: z.string() })` — the LLM returns a 0-1 score with an explanation. The reasoning (and any other schema fields) are available on each item via `scoreDetails` — see [Understanding Results](#understanding-results). For custom scoring dimensions, provide your own schema:
 
 ```typescript
 import { z } from 'zod';
@@ -184,9 +194,10 @@ npx axl-eval ./evals/qa.eval.ts                    # run a single file
 npx axl-eval ./evals/                               # run all *.eval.* files in a directory
 npx axl-eval ./evals/ --output ./results/v1.json    # save results to JSON
 npx axl-eval ./evals/ --config ./axl.config.ts      # use a specific runtime config
+npx axl-eval ./evals/ --conditions development      # add Node.js import conditions (monorepo source exports)
 ```
 
-The CLI resolves a runtime automatically: `--config <path>` > auto-detect `axl.config.*` > bare `new AxlRuntime()` (providers from env vars).
+The CLI resolves a runtime automatically: `--config <path>` > auto-detect `axl.config.*` > bare `new AxlRuntime()` (providers from env vars). Use `--conditions` when your eval file imports from monorepo packages that use conditional exports (e.g., `"development"` condition for source TypeScript instead of compiled dist).
 
 ### Programmatic
 
@@ -246,49 +257,97 @@ See the [@axlsdk/studio README](../axl-studio/README.md#lazy-eval-loading) for d
 
 ## Understanding Results
 
-Each eval run returns an `EvalResult` with per-item scores and aggregate statistics.
+Each eval run returns an `EvalResult` with per-item scores and aggregate statistics. Every item captures timing and cost alongside scores.
 
-**Scores** are `number | null` per scorer per item:
-- `0` to `1` — valid score from the scorer
-- `null` — the scorer failed (see `item.scorerErrors` for details)
+Each item has two ways to access scores:
+
+- **`item.scores`** — quick numeric lookup: `Record<string, number | null>`. Use this for simple checks and aggregation. `null` means the scorer failed (see `item.scorerErrors`).
+- **`item.scoreDetails`** — full context: `Record<string, ScorerDetail>`. Each detail has the numeric score plus `metadata` (e.g., LLM reasoning), per-scorer `duration`, and `cost`. Use this when you need to understand *why* a score is what it is.
 
 Summary statistics (mean, p50, p95, etc.) exclude `null` scores. If all scores for a scorer are `null`, the CLI shows `--` instead of misleading `0.00`.
 
 ```typescript
 const results = await runtime.eval({ ... });
 
-// Aggregate stats
+// ── Aggregate stats ──────────────────────────────────
 console.log(results.summary.scorers['quality'].mean);  // 0.85
 console.log(results.summary.count);                     // 50 items
 console.log(results.summary.failures);                  // 2 workflow errors
+console.log(results.summary.timing);                    // { mean, min, max, p50, p95 } in ms
+console.log(results.totalCost);                          // 0.42 (workflow + scorer LLM costs)
 
-// Per-item inspection
+// ── Per-item inspection ──────────────────────────────
 for (const item of results.items) {
   if (item.error) continue;                              // workflow threw
+
+  // Timing and cost
+  console.log(item.duration);                            // workflow execution ms
+  console.log(item.cost);                                // workflow LLM cost
+  console.log(item.scorerCost);                          // total scorer cost for this item
+
+  // Quick score access
+  console.log(item.scores['quality']);                    // 0.85 or null
+
+  // Rich per-scorer detail — reasoning, timing, cost
+  const detail = item.scoreDetails?.['quality'];
+  if (detail) {
+    console.log(detail.score);                           // 0.85
+    console.log(detail.metadata?.reasoning);             // "The answer is relevant..."
+    console.log(detail.duration);                        // scorer execution ms
+    console.log(detail.cost);                            // scorer LLM cost
+  }
+
+  // Error handling
   if (item.scores['quality'] === null) {
-    console.log('Scorer failed:', item.scorerErrors);    // e.g., ["Scorer "quality" threw: ..."]
+    console.log('Scorer failed:', item.scorerErrors);    // ["Scorer "quality" threw: ..."]
   }
 }
-
-// Cost includes both workflow and LLM scorer calls
-console.log(results.totalCost);  // 0.42
 ```
 
 ## Comparing Results
 
-Compare two runs to detect regressions and improvements:
+Compare two runs to detect regressions and improvements. Runs must use the same dataset and scorers.
 
 ```bash
 npx axl-eval compare ./results/v1.json ./results/v2.json
 npx axl-eval compare v1.json v2.json --fail-on-regression  # exit 1 if worse
 ```
 
+```
+Compare: baseline (3f8a2b1c) -> candidate (9d4e7f6a)
+
+  Scorer     Baseline  Candidate  Delta     Change
+  ──────────────────────────────────────────────────
+  quality       0.750      0.850    +0.100   +13.3%
+  safety        0.900      0.900    +0.000    +0.0%
+
+  Timing: baseline 2.10s -> candidate 4.30s (+104.8%)
+  Cost: baseline $0.45 -> candidate $0.31 (-31.1%)
+
+  Regressions: 1 | Improvements: 3 | Stable: 16
+```
+
+Programmatically:
+
 ```typescript
 import { evalCompare } from '@axlsdk/eval';
 
 const comparison = evalCompare(v1Results, v2Results);
-console.log(comparison.regressions);   // items that got worse
-console.log(comparison.improvements);  // items that got better
+
+// Score changes
+console.log(comparison.scorers.quality.delta);      // +0.1
+console.log(comparison.scorers.quality.deltaPercent); // +13.3
+
+// Timing and cost tradeoffs
+console.log(comparison.timing?.deltaPercent);  // +104.8 (slower)
+console.log(comparison.cost?.deltaPercent);    // -31.1 (cheaper)
+
+// Per-item regressions/improvements (delta > 0.1 threshold)
+for (const r of comparison.regressions) {
+  console.log(`Item ${r.itemIndex}: ${r.scorer} dropped ${r.baselineScore} → ${r.candidateScore}`);
+}
+
+console.log(comparison.summary);  // human-readable one-liner
 ```
 
 ## Eval Files in Detail
@@ -335,7 +394,26 @@ return { output, cost: ctx.totalCost };
 
 ### Common patterns
 
-**Budget** — cap cost per item with `runtime.createContext({ budget: '$0.50' })`.
+**Concurrency** — process items in parallel (default: 5):
+
+```typescript
+export default defineEval({
+  workflow: 'qa-eval',
+  dataset: ds,
+  scorers: [qualityJudge],
+  concurrency: 10,       // run 10 items in parallel
+  budget: '$5.00',        // stop if total cost exceeds $5
+});
+```
+
+**Per-item budget** — cap cost for a single workflow execution:
+
+```typescript
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const ctx = runtime.createContext({ budget: '$0.50' });
+  return { output: await ctx.ask(qaAgent, input.question) };
+}
+```
 
 **Timeout** — abort slow items:
 

package/dist/{chunk-OHZX3VFE.js → chunk-F2AALK5U.js}

@@ -35,6 +35,7 @@ function evalCompare(baseline, candidate) {
       const delta = round(cScore - bScore);
       if (delta < -0.1)
         regressions.push({
+          itemIndex: i,
           input: bItem.input,
           scorer: name,
           baselineScore: bScore,
@@ -43,6 +44,7 @@ function evalCompare(baseline, candidate) {
         });
       else if (delta > 0.1)
         improvements.push({
+          itemIndex: i,
           input: bItem.input,
           scorer: name,
           baselineScore: bScore,
@@ -51,6 +53,25 @@ function evalCompare(baseline, candidate) {
         });
     }
   }
+  let timing;
+  if (baseline.summary.timing && candidate.summary.timing) {
+    const bMean = baseline.summary.timing.mean;
+    const cMean = candidate.summary.timing.mean;
+    const delta = round(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
+    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+  }
+  let cost;
+  if (baseline.totalCost > 0 || candidate.totalCost > 0) {
+    const delta = round(candidate.totalCost - baseline.totalCost);
+    const deltaPercent = baseline.totalCost > 0 ? round(delta / baseline.totalCost * 100) : 0;
+    cost = {
+      baselineTotal: baseline.totalCost,
+      candidateTotal: candidate.totalCost,
+      delta,
+      deltaPercent
+    };
+  }
   const parts = [];
   for (const name of baselineScorerNames) {
     const s = scorers[name];
@@ -62,17 +83,40 @@ function evalCompare(baseline, candidate) {
       );
     }
   }
+  if (timing && Math.abs(timing.deltaPercent) > 1) {
+    const dir = timing.delta > 0 ? "slower" : "faster";
+    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
+  }
+  if (cost && Math.abs(cost.deltaPercent) > 1) {
+    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
+    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
+  }
   const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
   return {
     baseline: { id: baseline.id, metadata: baseline.metadata },
     candidate: { id: candidate.id, metadata: candidate.metadata },
     scorers,
+    timing,
+    cost,
     regressions,
     improvements,
     summary: summaryStr
   };
 }
 
+// src/scorer.ts
+function normalizeScorerResult(result) {
+  return typeof result === "number" ? { score: result } : result;
+}
+function scorer(config) {
+  return {
+    name: config.name,
+    description: config.description,
+    isLlm: false,
+    score: config.score
+  };
+}
+
 // src/runner.ts
 import { randomUUID } from "crypto";
 function computeStats(scores) {
@@ -109,18 +153,18 @@ async function runEval(config, executeWorkflow, runtime) {
       return runtime.resolveProvider(uri);
     }
   };
-  const evalItems = [];
+  const evalItems = new Array(items.length);
   let totalCost = 0;
   let budgetExceeded = false;
-  async function processItem(item) {
+  async function processItem(item, itemIndex) {
     if (budgetExceeded) {
-      evalItems.push({
+      evalItems[itemIndex] = {
         input: item.input,
         annotations: item.annotations,
         output: null,
         error: "Budget exceeded",
         scores: {}
-      });
+      };
       return;
     }
     const evalItem = {
@@ -129,64 +173,88 @@ async function runEval(config, executeWorkflow, runtime) {
       output: null,
       scores: {}
     };
+    const itemStart = Date.now();
     try {
       const result = await executeWorkflow(item.input, runtime);
+      evalItem.duration = Date.now() - itemStart;
       evalItem.output = result.output;
+      evalItem.cost = result.cost;
       if (result.cost != null) {
         totalCost += result.cost;
       }
     } catch (err) {
+      evalItem.duration = Date.now() - itemStart;
       evalItem.error = err instanceof Error ? err.message : String(err);
-      evalItems.push(evalItem);
+      evalItems[itemIndex] = evalItem;
       return;
     }
     if (budgetLimit != null && totalCost > budgetLimit) {
       budgetExceeded = true;
     }
-    for (const scorer of config.scorers) {
+    evalItem.scoreDetails = {};
+    let itemScorerCost = 0;
+    for (const scorer2 of config.scorers) {
+      const scorerStart = Date.now();
       try {
-        const score = await scorer.score(
+        const raw = await scorer2.score(
           evalItem.output,
           item.input,
           item.annotations,
           scorerContext
         );
-        if (scorer.isLlm) {
-          const lastCost = scorer._lastCost;
-          if (lastCost != null) {
-            totalCost += lastCost;
-          }
+        const scorerResult = normalizeScorerResult(raw);
+        if (scorerResult.cost != null) {
+          itemScorerCost += scorerResult.cost;
+          totalCost += scorerResult.cost;
         }
-        if (!Number.isFinite(score) || score < 0 || score > 1) {
+        const scorerDuration = Date.now() - scorerStart;
+        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
           if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
           evalItem.scorerErrors.push(
-            `Scorer "${scorer.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`
+            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
           );
-          evalItem.scores[scorer.name] = null;
+          evalItem.scores[scorer2.name] = null;
+          evalItem.scoreDetails[scorer2.name] = {
+            score: null,
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
         } else {
-          evalItem.scores[scorer.name] = round2(score);
+          evalItem.scores[scorer2.name] = round2(scorerResult.score);
+          evalItem.scoreDetails[scorer2.name] = {
+            score: round2(scorerResult.score),
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
         }
       } catch (err) {
-        if (scorer.isLlm) {
-          const lastCost = scorer._lastCost;
-          if (lastCost != null) {
-            totalCost += lastCost;
-          }
+        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
+        if (errCost != null) {
+          itemScorerCost += errCost;
+          totalCost += errCost;
         }
         if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
         evalItem.scorerErrors.push(
-          `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
         );
-        evalItem.scores[scorer.name] = null;
+        evalItem.scores[scorer2.name] = null;
+        evalItem.scoreDetails[scorer2.name] = {
+          score: null,
+          duration: Date.now() - scorerStart,
+          cost: errCost
+        };
       }
     }
-    evalItems.push(evalItem);
+    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    evalItems[itemIndex] = evalItem;
   }
   let index = 0;
   async function runNext() {
     while (index < items.length) {
       const currentIndex = index++;
-      await processItem(items[currentIndex]);
+      await processItem(items[currentIndex], currentIndex);
     }
   }
   const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
@@ -198,6 +266,8 @@ async function runEval(config, executeWorkflow, runtime) {
     const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
     scorerStats[name] = computeStats(scores);
   }
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
   return {
     id,
     workflow: config.workflow,
@@ -207,12 +277,14 @@ async function runEval(config, executeWorkflow, runtime) {
     totalCost,
     duration: Date.now() - startTime,
     items: evalItems,
-    summary: { count: items.length, failures, scorers: scorerStats }
+    summary: { count: items.length, failures, scorers: scorerStats, timing }
   };
 }
 
 export {
   evalCompare,
+  normalizeScorerResult,
+  scorer,
   runEval
 };
-//# sourceMappingURL=chunk-OHZX3VFE.js.map
+//# sourceMappingURL=chunk-F2AALK5U.js.map

package/dist/chunk-F2AALK5U.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/compare.ts","../src/scorer.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          itemIndex: i,\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          itemIndex: i,\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (use pre-computed summary.timing when available)\n  let timing: EvalComparison['timing'];\n  if (baseline.summary.timing && candidate.summary.timing) {\n    const bMean = baseline.summary.timing.mean;\n    const cMean = candidate.summary.timing.mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  // Cost comparison\n  let cost: EvalComparison['cost'];\n  if (baseline.totalCost > 0 || candidate.totalCost > 0) {\n    const delta = round(candidate.totalCost - baseline.totalCost);\n    const deltaPercent = baseline.totalCost > 0 ? round((delta / baseline.totalCost) * 100) : 0;\n    cost = {\n      baselineTotal: baseline.totalCost,\n      candidateTotal: candidate.totalCost,\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.output = result.output;\n      evalItem.cost = result.cost;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems[itemIndex] = evalItem;\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    evalItem.scoreDetails = {};\n    let itemScorerCost = 0;\n\n    for (const scorer of config.scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n          evalItem.scoreDetails[scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          evalItem.scores[scorer.name] = round(scorerResult.score);\n          evalItem.scoreDetails[scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        // Capture cost from error (LLM scorer attaches it)\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n        evalItem.scoreDetails[scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    evalItems[itemIndex] = evalItem;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats, timing },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,MAAI,SAAS,QAAQ,UAAU,UAAU,QAAQ,QAAQ;AACvD,UAAM,QAAQ,SAAS,QAAQ,OAAO;AACtC,UAAM,QAAQ,UAAU,QAAQ,OAAO;AACvC,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,aAAS,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EAC5E;AAGA,MAAI;AACJ,MAAI,SAAS,YAAY,KAAK,UAAU,YAAY,GAAG;AACrD,UAAM,QAAQ,MAAM,UAAU,YAAY,SAAS,SAAS;AAC5D,UAAM,eAAe,SAAS,YAAY,IAAI,MAAO,QAAQ,SAAS,YAAa,GAAG,IAAI;AAC1F,WAAO;AAAA,MACL,eAAe,SAAS;AAAA,MACxB,gBAAgB,UAAU;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC5GO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC3DA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWC,WAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAID,OAAM,aAAa,KAAK;AACvD,mBAAS,aAAaC,QAAO,IAAI,IAAI;AAAA,YACnC,OAAOD,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWC,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;","names":["round","scorer"]}