@axlsdk/eval 0.14.0 → 0.15.0

package/README.md

@@ -130,6 +130,8 @@ const lengthScore = scorer({
 });
 ```
 
+Scorers that return a non-finite value (`NaN` / `Infinity`) or a score outside `[0, 1]` are recorded as `null` with an entry in `item.scorerErrors` — they don't abort the run.
+
 ### LLM scorers
 
 Use an LLM as a judge. The scorer constructs a prompt from the input, output, and annotations, calls the LLM, and validates the response:
@@ -240,6 +242,36 @@ const results = await runEval(
 );
 ```
 
+**Optional `RunEvalOptions` 4th arg** (`onProgress` / `signal` / `captureTraces`):
+
+```typescript
+import type { RunEvalOptions, EvalProgressEvent } from '@axlsdk/eval';
+
+const controller = new AbortController();
+
+const results = await runEval(
+  { workflow: 'my-eval', dataset: ds, scorers: [...] },
+  executeWorkflow,
+  runtime,
+  {
+    onProgress: (event: EvalProgressEvent) => {
+      // `item_done` fires after each dataset item finishes (execution + scoring,
+      // or aborted/budget-exceeded). `run_done` fires once after all items.
+      // Narrow on `type` — `itemIndex` only exists on `item_done`.
+      if (event.type === 'item_done') {
+        console.log(`Item ${event.itemIndex + 1}/${event.totalItems} done`);
+      } else {
+        console.log(`All done: ${event.failures}/${event.totalItems} failed`);
+      }
+    },
+    signal: controller.signal,   // cancels between items (and between scorers within an item)
+    captureTraces: true,          // populates EvalItem.traces (success + failure paths)
+  },
+);
+```
+
+**Trust-boundary validation on workflow returns.** When your `executeWorkflow` callback returns `{ output, cost, metadata }`, the runner validates the untrusted fields before trusting them: `cost` must be a non-negative finite number, `metadata` must be a plain object (`Date`, `Map`, `Set`, class instances are rejected). Invalid values trigger a `console.warn` and fall back to trace-derived values from `runtime.trackExecution()`. A buggy workflow returning `{ cost: 'free' }` no longer silently NaN-poisons `totalCost`.
+
 ### Studio
 
 Eval files can be lazy-loaded by the Studio middleware for the Eval Runner panel:
@@ -308,6 +340,16 @@ for (const item of results.items) {
   if (item.scores['quality'] === null) {
     console.log('Scorer failed:', item.scorerErrors);    // ["Scorer "quality" threw: ..."]
   }
+
+  // Per-item trace events (when run with { captureTraces: true })
+  if (item.traces) {
+    console.log(`${item.traces.length} trace events captured`);
+    const agentCalls = item.traces.filter(e => e.type === 'agent_call');
+    console.log(`${agentCalls.length} LLM turns`);
+    // Failure-path traces are also populated (recovered from axlCapturedTraces
+    // on the thrown error) — captureTraces is especially useful for debugging
+    // items that error
+  }
 }
 ```
 
@@ -576,7 +618,7 @@ const comparison = evalCompare(baselineRuns, candidateRuns);
 | `scorer(config)` | Create a deterministic scorer |
 | `llmScorer(config)` | Create an LLM-as-judge scorer |
 | `defineEval(config)` | Wrap an eval config for CLI discovery |
-| `runEval(config, executeFn, runtime)` | Run an eval programmatically |
+| `runEval(config, executeFn, runtime, options?)` | Run an eval programmatically. `options: RunEvalOptions` accepts `onProgress` / `signal` / `captureTraces` |
 | `evalCompare(baseline, candidate, options?)` | Compare eval results with bootstrap CI |
 | `rescore(result, scorers, runtime, options?)` | Re-run scorers on saved outputs |
 | `aggregateRuns(runs)` | Aggregate multiple runs into mean ± std |
@@ -589,7 +631,7 @@ const comparison = evalCompare(baselineRuns, candidateRuns);
 |------|-------------|
 | `EvalConfig` | Eval definition (workflow, dataset, scorers, concurrency, budget) |
 | `EvalResult` | Full eval output (items, summary, cost, duration) |
-| `EvalItem` | Per-item result (input, output, scores, scoreDetails, metadata) |
+| `EvalItem` | Per-item result (input, output, scores, scoreDetails, metadata, traces?) |
 | `EvalSummary` | Aggregate statistics (count, failures, scorer stats, timing) |
 | `EvalComparison` | Comparison output (scorer deltas, CI, pRegression/pImprovement, n, regressions, improvements) |
 | `EvalCompareOptions` | Options for `evalCompare()` (`thresholds`) |
@@ -599,6 +641,8 @@ const comparison = evalCompare(baselineRuns, candidateRuns);
 | `RescoreOptions` | Options for `rescore()` (`concurrency`) |
 | `MultiRunSummary` | Aggregated multi-run output (per-scorer mean/std/min/max) |
 | `BootstrapCIResult` | CI result (`{ lower, upper, mean, pRegression, pImprovement }`) |
+| `RunEvalOptions` | Options for `runEval()` (`onProgress`, `signal`, `captureTraces`) |
+| `EvalProgressEvent` | Event passed to `onProgress`: `{ type: 'item_done', itemIndex, totalItems }` (emitted after each item finishes — success, failure, cancellation, or budget-exceeded) \| `{ type: 'run_done', totalItems, failures }` (emitted once after all items finish and stats are computed) |
 
 ## License
 

package/dist/{chunk-T4OFZQCJ.js → chunk-CV6VE5VT.js}

@@ -289,7 +289,36 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, runtime) {
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = randomUUID();
   const items = await config.dataset.getItems();
@@ -309,6 +338,17 @@ async function runEval(config, executeWorkflow, runtime) {
   let totalCost = 0;
   let budgetExceeded = false;
   async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = {
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Cancelled",
+        scores: {}
+      };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
     if (budgetExceeded) {
       evalItems[itemIndex] = {
         input: item.input,
@@ -317,6 +357,7 @@ async function runEval(config, executeWorkflow, runtime) {
         error: "Budget exceeded",
         scores: {}
       };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     const evalItem = {
@@ -327,18 +368,43 @@ async function runEval(config, executeWorkflow, runtime) {
     };
     const itemStart = Date.now();
     try {
-      const result = await executeWorkflow(item.input, runtime);
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.output = result.output;
-      evalItem.cost = result.cost;
-      if (result.metadata) evalItem.metadata = result.metadata;
-      if (result.cost != null) {
-        totalCost += result.cost;
+      if (options?.captureTraces) {
+        const tracked = await runtime.trackExecution(
+          async () => executeWorkflow(item.input, runtime),
+          { captureTraces: true }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = tracked.result.output;
+        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
+        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        if (tracked.traces && tracked.traces.length > 0) {
+          evalItem.traces = tracked.traces;
+        }
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
+      } else {
+        const result = await executeWorkflow(item.input, runtime);
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = result.output;
+        evalItem.cost = extractUserCost(result);
+        const meta = extractUserMetadata(result);
+        if (meta) evalItem.metadata = meta;
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
       }
     } catch (err) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.error = err instanceof Error ? err.message : String(err);
+      if (options?.captureTraces) {
+        const captured = err.axlCapturedTraces;
+        if (Array.isArray(captured) && captured.length > 0) {
+          evalItem.traces = captured;
+        }
+      }
       evalItems[itemIndex] = evalItem;
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     if (budgetLimit != null && totalCost > budgetLimit) {
@@ -347,6 +413,7 @@ async function runEval(config, executeWorkflow, runtime) {
     evalItem.scoreDetails = {};
     let itemScorerCost = 0;
     for (const scorer2 of config.scorers) {
+      if (options?.signal?.aborted) break;
       const scorerStart = Date.now();
       try {
         const raw = await scorer2.score(
@@ -402,6 +469,7 @@ async function runEval(config, executeWorkflow, runtime) {
     }
     evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
     evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
   let index = 0;
   async function runNext() {
@@ -475,6 +543,7 @@ async function runEval(config, executeWorkflow, runtime) {
     workflowsMeta.workflows = [config.workflow];
     workflowsMeta.workflowCounts = { [config.workflow]: items.length };
   }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
   return {
     id,
     dataset: config.dataset.name,
@@ -525,6 +594,7 @@ async function rescore(result, scorers, runtime, options) {
       annotations: original.annotations,
       output: original.output,
       metadata: original.metadata,
+      traces: original.traces,
       scores: {},
       scoreDetails: {}
     };
@@ -702,4 +772,4 @@ export {
   rescore,
   aggregateRuns
 };
-//# sourceMappingURL=chunk-T4OFZQCJ.js.map
+//# sourceMappingURL=chunk-CV6VE5VT.js.map

package/dist/chunk-CV6VE5VT.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/bootstrap.ts","../src/compare.ts","../src/scorer.ts","../src/utils.ts","../src/runner.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export type BootstrapCIResult = {\n  /** Lower bound of the confidence interval. */\n  lower: number;\n  /** Upper bound of the confidence interval. */\n  upper: number;\n  /** Mean of the original differences. */\n  mean: number;\n  /** Proportion of resamples where the mean difference is negative (probability of regression). */\n  pRegression: number;\n  /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n  pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n  differences: number[],\n  options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n  const n = differences.length;\n  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n  const nResamples = options?.nResamples ?? 1000;\n  const alpha = options?.alpha ?? 0.05;\n\n  const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n  if (n === 1)\n    return {\n      lower: differences[0],\n      upper: differences[0],\n      mean,\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  if (nResamples <= 0)\n    return {\n      lower: round(mean),\n      upper: round(mean),\n      mean: round(mean),\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  // Seeded or random number generator\n  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n  const resampleMeans: number[] = new Array(nResamples);\n  for (let r = 0; r < nResamples; r++) {\n    let sum = 0;\n    for (let i = 0; i < n; i++) {\n      sum += differences[Math.floor(rng() * n)];\n    }\n    resampleMeans[r] = sum / n;\n  }\n\n  resampleMeans.sort((a, b) => a - b);\n\n  const lowerIdx = Math.floor((alpha / 2) * nResamples);\n  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n  // Count resamples below/above zero for directional probabilities\n  // resampleMeans is already sorted, so binary search for zero\n  let belowZero = 0;\n  for (let i = 0; i < nResamples; i++) {\n    if (resampleMeans[i] < 0) belowZero++;\n    else break;\n  }\n  let aboveZero = 0;\n  for (let i = nResamples - 1; i >= 0; i--) {\n    if (resampleMeans[i] > 0) aboveZero++;\n    else break;\n  }\n\n  return {\n    lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n    mean: round(mean),\n    pRegression: round(belowZero / nResamples),\n    pImprovement: round(aboveZero / nResamples),\n  };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n  let state = seed | 0 || 1; // Ensure non-zero\n  return () => {\n    state ^= state << 13;\n    state ^= state >> 17;\n    state ^= state << 5;\n    return (state >>> 0) / 0x100000000;\n  };\n}\n","import type {\n  EvalResult,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n  scorerName: string,\n  options: EvalCompareOptions | undefined,\n  metadata: Record<string, unknown>,\n): number {\n  // Explicit global threshold\n  if (typeof options?.thresholds === 'number') return options.thresholds;\n\n  // Per-scorer threshold map\n  if (options?.thresholds && typeof options.thresholds === 'object') {\n    const perScorer = options.thresholds[scorerName];\n    if (perScorer != null) return perScorer;\n    // Fall through to auto-calibration if scorer not in map\n  }\n\n  // Auto-calibrate from scorerTypes metadata\n  const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n  if (scorerTypes && scorerName in scorerTypes) {\n    return scorerTypes[scorerName] === 'llm'\n      ? DEFAULT_LLM_THRESHOLD\n      : DEFAULT_DETERMINISTIC_THRESHOLD;\n  }\n\n  // Legacy fallback for results without scorerTypes\n  return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n  baseline: EvalResult | EvalResult[],\n  candidate: EvalResult | EvalResult[],\n  options?: EvalCompareOptions,\n): EvalComparison {\n  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n  if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n    throw new Error('Cannot compare empty eval result arrays');\n  }\n\n  // Use the first run as representative for metadata, dataset, scorers\n  const baselineRef = baselineRuns[0];\n  const candidateRef = candidateRuns[0];\n\n  if (baselineRef.dataset !== candidateRef.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n  // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n  const pairedDiffs: Record<string, number[]> = {};\n  for (const name of baselineScorerNames) {\n    pairedDiffs[name] = [];\n  }\n  const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n  for (let r = 0; r < runCount; r++) {\n    const bRun = baselineRuns[r];\n    const cRun = candidateRuns[r];\n    const minLength = Math.min(bRun.items.length, cRun.items.length);\n    for (let i = 0; i < minLength; i++) {\n      const bItem = bRun.items[i];\n      const cItem = cRun.items[i];\n      if (bItem.error || cItem.error) continue;\n      for (const name of baselineScorerNames) {\n        const bScore = bItem.scores[name];\n        const cScore = cItem.scores[name];\n        if (bScore != null && cScore != null) {\n          pairedDiffs[name].push(cScore - bScore);\n        }\n      }\n    }\n  }\n\n  // Compute aggregate means across runs\n  const baselineMeans: Record<string, number> = {};\n  const candidateMeans: Record<string, number> = {};\n  for (const name of baselineScorerNames) {\n    baselineMeans[name] =\n      baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      baselineRuns.length;\n    candidateMeans[name] =\n      candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      candidateRuns.length;\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMeanRaw = baselineMeans[name];\n    const cMeanRaw = candidateMeans[name];\n    const deltaRaw = cMeanRaw - bMeanRaw;\n    const bMean = round(bMeanRaw);\n    const cMean = round(cMeanRaw);\n    const delta = round(deltaRaw);\n    const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n    const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n    const entry: EvalComparison['scorers'][string] = {\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      deltaPercent,\n    };\n\n    // Compute bootstrap CI when we have enough paired data\n    const diffs = pairedDiffs[name];\n    if (diffs.length >= 2) {\n      const ci = pairedBootstrapCI(diffs);\n      entry.ci = { lower: ci.lower, upper: ci.upper };\n      entry.pRegression = ci.pRegression;\n      entry.pImprovement = ci.pImprovement;\n      entry.n = diffs.length;\n      // Significant when CI excludes zero AND delta exceeds practical threshold\n      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n    }\n\n    scorers[name] = entry;\n  }\n\n  // Per-item regressions/improvements.\n  // For multi-run: average each item's score across runs to get a stable per-item comparison.\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const itemCount = Math.min(\n    ...baselineRuns.map((r) => r.items.length),\n    ...candidateRuns.map((r) => r.items.length),\n  );\n\n  for (let i = 0; i < itemCount; i++) {\n    // Check if any run has an error for this item\n    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n    if (hasBaselineError || hasCandidateError) continue;\n\n    for (const name of baselineScorerNames) {\n      // Average this item's score across all runs on each side\n      const bScores = baselineRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      const cScores = candidateRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      if (bScores.length === 0 || cScores.length === 0) continue;\n\n      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n      const delta = round(cAvg - bAvg);\n      const threshold = resolveThreshold(name, options, baselineRef.metadata);\n      if (delta < -threshold)\n        regressions.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n      else if (delta > threshold)\n        improvements.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (average across all runs for multi-run)\n  let timing: EvalComparison['timing'];\n  const baselineTimings = baselineRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  const candidateTimings = candidateRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n  }\n\n  // Cost comparison (per-run average for multi-run)\n  let cost: EvalComparison['cost'];\n  const baselineAvgCost =\n    baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n  const candidateAvgCost =\n    candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n  if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n    const deltaRaw = candidateAvgCost - baselineAvgCost;\n    const delta = round(deltaRaw);\n    const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n    cost = {\n      baselineTotal: round(baselineAvgCost),\n      candidateTotal: round(candidateAvgCost),\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      let sigLabel = '';\n      if (s.significant === true) sigLabel = ' (significant)';\n      else if (s.significant === false) sigLabel = ' (not significant)';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","export function computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary, RunEvalOptions } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/**\n * Extract a user-returned cost only if it's a non-negative finite number.\n * Guards against workflows that return `{ cost: 'free' }`, `{ cost: NaN }`,\n * `{ cost: -1 }`, `{ cost: Infinity }`, etc. — the TS type says `cost?: number`\n * but at runtime we can't trust that.\n *\n * Negative costs are rejected because (a) cost is a USD amount and negative\n * values are nonsensical, and (b) a negative cost would silently shrink\n * `totalCost` below the budget limit check at the processItem level,\n * letting a buggy/malicious workflow run unbounded. `0` is preserved\n * because free operations on paid models are valid.\n *\n * When a user-supplied value is rejected (present but invalid), we log\n * once per item via `console.warn` so the type violation is visible\n * rather than silently swallowed.\n */\nfunction extractUserCost(result: unknown, label = 'executeWorkflow'): number | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const raw = (result as { cost?: unknown }).cost;\n  if (raw === undefined) return undefined;\n  if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return raw;\n\n  console.warn(\n    `[axl-eval] Ignoring invalid \\`cost\\` from ${label} return: expected non-negative finite number, got ${typeof raw === 'number' ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`,\n  );\n  return undefined;\n}\n\n/**\n * Extract a user-returned metadata record only if it's a plain object.\n * Rejects arrays, null, scalars, and exotic object types (Date, Map, Set,\n * Error, class instances) that would satisfy a loose `typeof === 'object'`\n * check but break `Record<string, unknown>` assumptions in downstream\n * consumers (spread, Object.entries, property access).\n */\nfunction extractUserMetadata(\n  result: unknown,\n  label = 'executeWorkflow',\n): Record<string, unknown> | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const meta = (result as { metadata?: unknown }).metadata;\n  if (meta === undefined) return undefined;\n  if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${meta === null ? 'null' : Array.isArray(meta) ? 'array' : typeof meta}.`,\n    );\n    return undefined;\n  }\n  // Reject exotic objects (Date, Map, Set, Error, class instances) whose\n  // prototype chain differs from Object.prototype. Those pass `typeof ===\n  // 'object'` but don't behave like `Record<string, unknown>`.\n  const proto = Object.getPrototypeOf(meta);\n  if (proto !== Object.prototype && proto !== null) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${(proto?.constructor?.name as string | undefined) ?? 'exotic object'}.`,\n    );\n    return undefined;\n  }\n  return meta as Record<string, unknown>;\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n  runtime: AxlRuntime,\n  options?: RunEvalOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (options?.signal?.aborted) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Cancelled',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      // When captureTraces is on, wrap the user's executeWorkflow in\n      // trackExecution so we get a per-item TraceEvent[] scoped via\n      // AsyncLocalStorage. trackExecution's scope walks the parent chain, so\n      // any nested trackExecution (e.g. from the CLI) continues to work.\n      if (options?.captureTraces) {\n        const tracked = await runtime.trackExecution(\n          async () => executeWorkflow(item.input, runtime),\n          { captureTraces: true },\n        );\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = tracked.result.output;\n        // Prefer user-returned cost/metadata; fall back to tracked values.\n        // Type-guarded so a non-number cost (e.g. `{ cost: 'free' }`) or a\n        // non-object metadata (e.g. an array or scalar) doesn't silently\n        // corrupt downstream math or shape expectations.\n        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;\n        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;\n        if (tracked.traces && tracked.traces.length > 0) {\n          evalItem.traces = tracked.traces;\n        }\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      } else {\n        const result = await executeWorkflow(item.input, runtime);\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = result.output;\n        evalItem.cost = extractUserCost(result);\n        const meta = extractUserMetadata(result);\n        if (meta) evalItem.metadata = meta;\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      // Failed items are exactly when per-item traces are most valuable —\n      // recover them from the captured-traces side-channel `trackExecution`\n      // attaches to the error on the failure path.\n      if (options?.captureTraces) {\n        const captured = (err as { axlCapturedTraces?: unknown }).axlCapturedTraces;\n        if (Array.isArray(captured) && captured.length > 0) {\n          evalItem.traces = captured as EvalItem['traces'];\n        }\n      }\n      evalItems[itemIndex] = evalItem;\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    evalItem.scoreDetails = {};\n    let itemScorerCost = 0;\n\n    for (const scorer of config.scorers) {\n      if (options?.signal?.aborted) break;\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n          evalItem.scoreDetails[scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          evalItem.scores[scorer.name] = round(scorerResult.score);\n          evalItem.scoreDetails[scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        // Capture cost from error (LLM scorer attaches it)\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n        evalItem.scoreDetails[scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    evalItems[itemIndex] = evalItem;\n    options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of config.scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  // Aggregate per-model LLM call counts across all items\n  const totalModelCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.modelCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n      const itemModels = item.metadata?.models;\n      if (Array.isArray(itemModels)) {\n        for (const m of itemModels) {\n          if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // models: unique list sorted by total calls (most-called first)\n  // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n  const modelsMeta: Record<string, unknown> = {};\n  if (totalModelCalls.size > 0) {\n    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n    modelsMeta.models = sorted.map(([m]) => m);\n    modelsMeta.modelCounts = Object.fromEntries(sorted);\n  }\n\n  // Aggregate per-workflow call counts across all items (parallel to models).\n  // Workflows come from trackExecution's trace-event collection — callers\n  // don't specify workflow names anywhere, they just appear because the\n  // runtime emits workflow_start events for every execute() call.\n  const totalWorkflowCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.workflowCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique workflows per item\n      const itemWorkflows = item.metadata?.workflows;\n      if (Array.isArray(itemWorkflows)) {\n        for (const w of itemWorkflows) {\n          if (typeof w === 'string')\n            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // Fall back to config.workflow when the callback bypassed the runtime's\n  // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n  // metadata.workflows array non-empty for the common test-harness case.\n  const workflowsMeta: Record<string, unknown> = {};\n  if (totalWorkflowCalls.size > 0) {\n    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n    workflowsMeta.workflows = sorted.map(([w]) => w);\n    workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n  } else if (config.workflow) {\n    workflowsMeta.workflows = [config.workflow];\n    workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n  }\n\n  options?.onProgress?.({ type: 'run_done', totalItems: items.length, failures });\n\n  return {\n    id,\n    dataset: config.dataset.name,\n    metadata: {\n      ...config.metadata,\n      scorerTypes,\n      ...modelsMeta,\n      ...workflowsMeta,\n    },\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats, timing },\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n  concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n  result: EvalResult,\n  scorers: Scorer[],\n  runtime: AxlRuntime,\n  options?: RescoreOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const concurrency = options?.concurrency ?? 5;\n\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const rescored: EvalItem[] = new Array(result.items.length);\n  let totalCost = 0;\n\n  async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n    // Pass through error items without scoring\n    if (original.error) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: original.error,\n        scores: {},\n      };\n      return;\n    }\n\n    // Preserve per-item traces from the original run: rescore re-runs scorers\n    // but leaves the workflow untouched, so the original execution traces\n    // remain accurate and useful for diagnosing score changes.\n    const item: EvalItem = {\n      input: original.input,\n      annotations: original.annotations,\n      output: original.output,\n      metadata: original.metadata,\n      traces: original.traces,\n      scores: {},\n      scoreDetails: {},\n    };\n\n    let itemScorerCost = 0;\n\n    for (const scorer of scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!item.scorerErrors) item.scorerErrors = [];\n          item.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          item.scores[scorer.name] = null;\n          item.scoreDetails![scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          item.scores[scorer.name] = round(scorerResult.score);\n          item.scoreDetails![scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!item.scorerErrors) item.scorerErrors = [];\n        item.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        item.scores[scorer.name] = null;\n        item.scoreDetails![scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    rescored[itemIndex] = item;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < result.items.length) {\n      const currentIndex = index++;\n      await rescoreItem(result.items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n    runNext(),\n  );\n  await Promise.all(workers);\n\n  const failures = rescored.filter((i) => i.error).length;\n  const scorerNames = scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = rescored\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  return {\n    id: randomUUID(),\n    dataset: result.dataset,\n    metadata: (() => {\n      // Strip run group membership — rescored results are independent evaluations.\n      // metadata.workflows is preserved via ...rest so the rescored result keeps\n      // the same workflow attribution as the original.\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n      const merged: Record<string, unknown> = {\n        ...rest,\n        rescored: true,\n        originalId: result.id,\n        scorerTypes,\n      };\n      // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n      // as a top-level string field with no `metadata.workflows`. Migrate it\n      // forward so rescored results from old artifacts retain their workflow\n      // attribution under the modern shape.\n      if (!Array.isArray(merged.workflows)) {\n        const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n        if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n          merged.workflows = [legacyWorkflow];\n          merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n        }\n      }\n      return merged;\n    })(),\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: rescored,\n    summary: { count: result.items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n  runGroupId: string;\n  runCount: number;\n  /**\n   * Workflow names observed across all runs in the group, deduped and ordered\n   * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n   * multi-run group produced by `axl-eval --runs N` typically has one entry;\n   * heterogeneous groups (custom callbacks) can have multiple.\n   */\n  workflows: string[];\n  dataset: string;\n  totalCost: number;\n  totalDuration: number;\n  scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n  timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n  if (values.length <= 1) return 0;\n  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n  return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n  if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n  const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n  const { dataset } = runs[0];\n\n  // Union workflow names across all runs in the group, first-seen first.\n  // Most groups have one, but custom callbacks can produce heterogeneous ones.\n  const seenWorkflows = new Set<string>();\n  const workflows: string[] = [];\n  for (const run of runs) {\n    const list = run.metadata.workflows;\n    if (Array.isArray(list)) {\n      for (const w of list) {\n        if (typeof w === 'string' && !seenWorkflows.has(w)) {\n          seenWorkflows.add(w);\n          workflows.push(w);\n        }\n      }\n    }\n  }\n\n  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n  // Collect per-scorer means from each run\n  const scorerNames = Object.keys(runs[0].summary.scorers);\n  const scorers: MultiRunSummary['scorers'] = {};\n\n  for (const name of scorerNames) {\n    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n    const minMean = Math.min(...means);\n    const maxMean = Math.max(...means);\n\n    scorers[name] = {\n      mean: round(meanOfMeans),\n      std: round(std(means, meanOfMeans)),\n      min: round(minMean),\n      max: round(maxMean),\n    };\n  }\n\n  // Timing aggregation\n  let timing: MultiRunSummary['timing'];\n  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n  if (timingMeans.length === runs.length && timingMeans.length > 0) {\n    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n    timing = {\n      mean: round(meanTiming),\n      std: round(std(timingMeans, meanTiming)),\n    };\n  }\n\n  return {\n    runGroupId,\n    runCount: runs.length,\n    workflows,\n    dataset,\n    totalCost: round(totalCost),\n    totalDuration: round(totalDuration),\n    scorers,\n    timing,\n  };\n}\n"],"mappings":";AAmBO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAO,MAAM,IAAI;AAAA,MACjB,OAAO,MAAM,IAAI;AAAA,MACjB,MAAM,MAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAO,MAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAM,MAAM,IAAI;AAAA,IAChB,aAAa,MAAM,YAAY,UAAU;AAAA,IACzC,cAAc,MAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9PO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC/DO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEO,SAASA,OAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,SAAS,kBAAkB;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAkBA,SAAS,gBAAgB,QAAiB,QAAQ,mBAAuC;AACvF,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,MAAO,OAA8B;AAC3C,MAAI,QAAQ,OAAW,QAAO;AAC9B,MAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,KAAK,OAAO,EAAG,QAAO;AAExE,UAAQ;AAAA,IACN,6CAA6C,KAAK,qDAAqD,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI,OAAO,GAAG;AAAA,EAC3J;AACA,SAAO;AACT;AASA,SAAS,oBACP,QACA,QAAQ,mBAC6B;AACrC,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,OAAQ,OAAkC;AAChD,MAAI,SAAS,OAAW,QAAO;AAC/B,MAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;AACpE,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAuC,SAAS,OAAO,SAAS,MAAM,QAAQ,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,IACnK;AACA,WAAO;AAAA,EACT;AAIA,QAAM,QAAQ,OAAO,eAAe,IAAI;AACxC,MAAI,UAAU,OAAO,aAAa,UAAU,MAAM;AAChD,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAwC,OAAO,aAAa,QAA+B,eAAe;AAAA,IAClK;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAsB,QACpB,QACA,iBAIA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,SAAS,QAAQ,SAAS;AAC5B,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAKF,UAAI,SAAS,eAAe;AAC1B,cAAM,UAAU,MAAM,QAAQ;AAAA,UAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,UAC/C,EAAE,eAAe,KAAK;AAAA,QACxB;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,QAAQ,OAAO;AAKjC,iBAAS,OAAO,gBAAgB,QAAQ,MAAM,KAAK,QAAQ;AAC3D,iBAAS,WAAW,oBAAoB,QAAQ,MAAM,KAAK,QAAQ;AACnE,YAAI,QAAQ,UAAU,QAAQ,OAAO,SAAS,GAAG;AAC/C,mBAAS,SAAS,QAAQ;AAAA,QAC5B;AACA,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF,OAAO;AACL,cAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,OAAO;AACzB,iBAAS,OAAO,gBAAgB,MAAM;AACtC,cAAM,OAAO,oBAAoB,MAAM;AACvC,YAAI,KAAM,UAAS,WAAW;AAC9B,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAIhE,UAAI,SAAS,eAAe;AAC1B,cAAM,WAAY,IAAwC;AAC1D,YAAI,MAAM,QAAQ,QAAQ,KAAK,SAAS,SAAS,GAAG;AAClD,mBAAS,SAAS;AAAA,QACpB;AAAA,MACF;AACA,gBAAU,SAAS,IAAI;AACvB,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWC,WAAU,OAAO,SAAS;AACnC,UAAI,SAAS,QAAQ,QAAS;AAC9B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACvD,mBAAS,aAAaD,QAAO,IAAI,IAAI;AAAA,YACnC,OAAOC,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWD,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AACvB,aAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAAA,EAClF;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,WAAS,aAAa,EAAE,MAAM,YAAY,YAAY,MAAM,QAAQ,SAAS,CAAC;AAE9E,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;ACpXA,SAAS,cAAAE,mBAAkB;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAKA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,SAAS;AAAA,MACjB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAWC,WAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAOA,QAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACnD,eAAK,aAAcD,QAAO,IAAI,IAAI;AAAA,YAChC,OAAOC,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAWD,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,aAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,IAAID,YAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAKf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;AC5LA,SAAS,cAAAG,mBAAkB;AAoB3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,cAAyBC,YAAW;AACzE,QAAM,EAAE,QAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAMC,OAAM,WAAW;AAAA,MACvB,KAAKA,OAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAKA,OAAM,OAAO;AAAA,MAClB,KAAKA,OAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAMA,OAAM,UAAU;AAAA,MACtB,KAAKA,OAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA,WAAWA,OAAM,SAAS;AAAA,IAC1B,eAAeA,OAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["round","round","scorer","round","randomUUID","scorer","round","randomUUID","randomUUID","round"]}

package/dist/cli.cjs

@@ -311,7 +311,36 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, runtime) {
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
@@ -331,6 +360,17 @@ async function runEval(config, executeWorkflow, runtime) {
   let totalCost = 0;
   let budgetExceeded = false;
   async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = {
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Cancelled",
+        scores: {}
+      };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
     if (budgetExceeded) {
       evalItems[itemIndex] = {
         input: item.input,
@@ -339,6 +379,7 @@ async function runEval(config, executeWorkflow, runtime) {
         error: "Budget exceeded",
         scores: {}
       };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     const evalItem = {
@@ -349,18 +390,43 @@ async function runEval(config, executeWorkflow, runtime) {
     };
     const itemStart = Date.now();
     try {
-      const result = await executeWorkflow(item.input, runtime);
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.output = result.output;
-      evalItem.cost = result.cost;
-      if (result.metadata) evalItem.metadata = result.metadata;
-      if (result.cost != null) {
-        totalCost += result.cost;
+      if (options?.captureTraces) {
+        const tracked = await runtime.trackExecution(
+          async () => executeWorkflow(item.input, runtime),
+          { captureTraces: true }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = tracked.result.output;
+        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
+        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        if (tracked.traces && tracked.traces.length > 0) {
+          evalItem.traces = tracked.traces;
+        }
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
+      } else {
+        const result = await executeWorkflow(item.input, runtime);
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = result.output;
+        evalItem.cost = extractUserCost(result);
+        const meta = extractUserMetadata(result);
+        if (meta) evalItem.metadata = meta;
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
       }
     } catch (err) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.error = err instanceof Error ? err.message : String(err);
+      if (options?.captureTraces) {
+        const captured = err.axlCapturedTraces;
+        if (Array.isArray(captured) && captured.length > 0) {
+          evalItem.traces = captured;
+        }
+      }
       evalItems[itemIndex] = evalItem;
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     if (budgetLimit != null && totalCost > budgetLimit) {
@@ -369,6 +435,7 @@ async function runEval(config, executeWorkflow, runtime) {
     evalItem.scoreDetails = {};
     let itemScorerCost = 0;
     for (const scorer of config.scorers) {
+      if (options?.signal?.aborted) break;
       const scorerStart = Date.now();
       try {
         const raw = await scorer.score(
@@ -424,6 +491,7 @@ async function runEval(config, executeWorkflow, runtime) {
     }
     evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
     evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
   let index = 0;
   async function runNext() {
@@ -497,6 +565,7 @@ async function runEval(config, executeWorkflow, runtime) {
     workflowsMeta.workflows = [config.workflow];
     workflowsMeta.workflowCounts = { [config.workflow]: items.length };
   }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
   return {
     id,
     dataset: config.dataset.name,
@@ -547,6 +616,7 @@ async function rescore(result, scorers, runtime, options) {
       annotations: original.annotations,
       output: original.output,
       metadata: original.metadata,
+      traces: original.traces,
       scores: {},
       scoreDetails: {}
     };
@@ -785,7 +855,8 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--conditions",
   "--fail-on-regression",
   "--threshold",
-  "--runs"
+  "--runs",
+  "--capture-traces"
 ]);
 async function main() {
   const args = process.argv.slice(2);
@@ -797,6 +868,10 @@ Usage:
   axl-eval <path> --output <file>         Save results to JSON
   axl-eval <path> --config <file>         Use config file for runtime
   axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval <path> --capture-traces        Populate EvalItem.traces on every item
+                                          (success + failure). Adds memory
+                                          overhead proportional to dataset size
+                                          x turns x agents; off by default.
   axl-eval rescore <results> <eval-file>  Re-run scorers on saved outputs
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
@@ -1117,6 +1192,7 @@ function parseEvalArgs(args) {
   let configArg;
   let conditions = [];
   let runs = 1;
+  let captureTraces = false;
   const paths = [];
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -1131,6 +1207,8 @@ function parseEvalArgs(args) {
       else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
       else
         conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
+    } else if (arg === "--capture-traces") {
+      captureTraces = true;
     } else if (arg.startsWith("--")) {
       if (!KNOWN_FLAGS.has(arg)) {
         console.error(`Unknown flag: ${arg}`);
@@ -1140,10 +1218,10 @@ function parseEvalArgs(args) {
       paths.push(arg);
     }
   }
-  return { outputPath, configArg, conditions, runs, paths };
+  return { outputPath, configArg, conditions, runs, captureTraces, paths };
 }
 async function runEvalCommand(args) {
-  const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);
+  const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -1198,13 +1276,14 @@ async function runEvalCommand(args) {
           );
           executeWorkflow = async (input) => ({ output: input });
         }
+        const runOptions = captureTraces ? { captureTraces: true } : void 0;
         if (runs > 1) {
           const { randomUUID: randomUUID4 } = await import("crypto");
           const runGroupId = randomUUID4();
           const runResults = [];
           for (let r = 0; r < runs; r++) {
             console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
-            const result = await runEval(evalConfig, executeWorkflow, runtime);
+            const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
             result.metadata.runGroupId = runGroupId;
             result.metadata.runIndex = r;
             runResults.push(result);
@@ -1213,7 +1292,7 @@ async function runEvalCommand(args) {
           const summary = aggregateRuns(runResults);
           console.log("\n" + formatMultiRunTable(summary) + "\n");
         } else {
-          const result = await runEval(evalConfig, executeWorkflow, runtime);
+          const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
         }