@axlsdk/eval 0.22.3 → 0.23.1

package/README.md

@@ -427,7 +427,8 @@ console.log(results.summary.scorers['quality'].failed); // 2  (scorer ran and th
 console.log(results.summary.scorers['quality'].skipped); // 0  (items the applies predicate skipped)
 console.log(results.summary.count);                     // 50 items
 console.log(results.summary.failures);                  // 2 workflow errors
-console.log(results.summary.timing);                    // { mean, min, max, p50, p95 } in ms
+console.log(results.summary.timing);                    // { mean, min, max, p50, p95 } in ms — WORKFLOW wall clock
+console.log(results.summary.modelTiming);               // per model: { calls, wireMs, queuedMs, retryMs, firstTokenMs? } — PROVIDER latency, per-CALL distributions
 console.log(results.totalCost);                          // 0.42 (workflow + scorer LLM costs)
 console.log(results.metadata.models);                    // ["openai:gpt-4o"] (sorted by usage)
 console.log(results.metadata.modelCounts);               // { "openai:gpt-4o": 48, "openai:gpt-4o-mini": 2 } (total LLM calls per model)
@@ -437,7 +438,8 @@ for (const item of results.items) {
   if (item.error) continue;                              // workflow threw
 
   // Timing and cost
-  console.log(item.duration);                            // workflow execution ms
+  console.log(item.duration);                            // workflow execution ms (tools, gates, queue and all)
+  console.log(item.timing);                              // per model: { calls, queuedMs, retryMs, wireMs, firstTokenMs?, firstTokenCalls? } sums — absent if nothing was timed
   console.log(item.cost);                                // workflow LLM cost
   console.log(item.scorerCost);                          // total scorer cost for this item
 
@@ -829,8 +831,9 @@ const comparison = evalCompare(baselineRuns, candidateRuns);
 |------|-------------|
 | `EvalConfig` | Eval definition (workflow, dataset, scorers, concurrency, scorerConcurrency, budget) |
 | `EvalResult` | Full eval output (items, summary, cost, duration) |
-| `EvalItem` | Per-item result (input, output, scores, scoreDetails, metadata, traces?) |
-| `EvalSummary` | Aggregate statistics (count, failures, per-scorer stats incl. `scored`/`failed`/`skipped`, timing) |
+| `EvalItem` | Per-item result (input, output, scores, scoreDetails, metadata, traces?, `timing?`) |
+| `EvalSummary` | Aggregate statistics (count, failures, per-scorer stats incl. `scored`/`failed`/`skipped`, wall-clock `timing`, per-model `modelTiming?`) |
+| `ItemModelTiming` / `ModelTimingStats` | Per-model provider latency. `EvalItem.timing` carries compact per-item sums; `EvalSummary.modelTiming` carries `{ mean, min, max, p50, p95 }` distributions over PER-CALL values, so one provider call is one sample. Only SUCCESSFUL timed calls are sampled, so the stats describe answers rather than a blend of answers and failures. Compare models on `wireMs` / `firstTokenMs`; `queuedMs` is Axl's own rate-limiter wait and `retryMs` is the provider's throttling, so neither distorts the comparison. Absent when nothing reported timing |
 | `EvalComparison` | Comparison output (scorer deltas, CI, pRegression/pImprovement, n, per-side `runCount` / `partial?`, regressions, improvements). Both sides truncate to `min(baseline.length, candidate.length)` so means align with the paired bootstrap CI's sample |
 | `EvalComparisonPartial` | `{ completed, attempted }` set on a comparison side when the pooled run count is less than the original batch's planned count |
 | `EvalCompareOptions` | Options for `evalCompare()` (`thresholds`) |

package/dist/{chunk-C7CICWIA.js → chunk-FQHKBSZD.js}

@@ -515,6 +515,27 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const evalItems = new Array(items.length);
   let totalCost = 0;
   let budgetExceeded = false;
+  const runCallSamples = /* @__PURE__ */ new Map();
+  function absorbTiming(rollup, evalItem) {
+    if (!rollup) return;
+    const perItem = {};
+    for (const [model, bucket] of Object.entries(rollup)) {
+      const sums = {
+        calls: bucket.calls,
+        queuedMs: bucket.queuedMs,
+        retryMs: bucket.retryMs,
+        wireMs: bucket.wireMs,
+        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
+      };
+      perItem[model] = sums;
+      const samples = bucket.samples;
+      if (!samples) continue;
+      const pooled = runCallSamples.get(model) ?? [];
+      for (const sample of samples) pooled.push(sample);
+      runCallSamples.set(model, pooled);
+    }
+    evalItem.timing = perItem;
+  }
   async function processItem(item, itemIndex) {
     if (options?.signal?.aborted) {
       evalItems[itemIndex] = {
@@ -549,13 +570,14 @@ async function runEval(config, executeWorkflow, runtime, options) {
       if (options?.captureTraces) {
         const tracked = await runtime.trackExecution(
           async () => executeWorkflow(item.input, runtime),
-          { captureTraces: true }
+          { captureTraces: true, captureTimingSamples: true }
         );
         evalItem.duration = Date.now() - itemStart;
         evalItem.output = tracked.result.output;
         evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
         if (tracked.unpriced) evalItem.unpriced = true;
         evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        absorbTiming(tracked.modelTiming, evalItem);
         if (tracked.traces && tracked.traces.length > 0) {
           evalItem.traces = tracked.traces;
         }
@@ -563,12 +585,24 @@ async function runEval(config, executeWorkflow, runtime, options) {
           totalCost += evalItem.cost;
         }
       } else {
-        const result = await executeWorkflow(item.input, runtime);
+        let trackedTiming;
+        let result;
+        if (typeof runtime.trackExecution === "function") {
+          const tracked = await runtime.trackExecution(
+            async () => executeWorkflow(item.input, runtime),
+            { captureTraces: false, captureTimingSamples: true }
+          );
+          result = tracked.result;
+          trackedTiming = tracked.modelTiming;
+        } else {
+          result = await executeWorkflow(item.input, runtime);
+        }
         evalItem.duration = Date.now() - itemStart;
         evalItem.output = result.output;
         evalItem.cost = extractUserCost(result);
         const meta = extractUserMetadata(result);
         if (meta) evalItem.metadata = meta;
+        absorbTiming(trackedTiming, evalItem);
         if (evalItem.cost != null) {
           totalCost += evalItem.cost;
         }
@@ -636,6 +670,19 @@ async function runEval(config, executeWorkflow, runtime, options) {
   }
   const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
   const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const modelTiming = runCallSamples.size > 0 ? Object.fromEntries(
+    [...runCallSamples.entries()].map(([model, samples]) => {
+      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
+      const stats = {
+        calls: samples.length,
+        wireMs: computeStats(samples.map((s) => s.wireMs)),
+        queuedMs: computeStats(samples.map((s) => s.queuedMs)),
+        retryMs: computeStats(samples.map((s) => s.retryMs)),
+        ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
+      };
+      return [model, stats];
+    })
+  ) : void 0;
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -707,6 +754,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
       failures,
       scorers: scorerStats,
       timing,
+      ...modelTiming ? { modelTiming } : {},
       ...degraded ? { degraded } : {}
     }
   };
@@ -901,4 +949,4 @@ export {
   rescore,
   aggregateRuns
 };
-//# sourceMappingURL=chunk-C7CICWIA.js.map
+//# sourceMappingURL=chunk-FQHKBSZD.js.map

package/dist/chunk-FQHKBSZD.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/bootstrap.ts","../src/utils.ts","../src/compare.ts","../src/scorer.ts","../src/score-item.ts","../src/runner.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export type BootstrapCIResult = {\n  /** Lower bound of the confidence interval. */\n  lower: number;\n  /** Upper bound of the confidence interval. */\n  upper: number;\n  /** Mean of the original differences. */\n  mean: number;\n  /** Proportion of resamples where the mean difference is negative (probability of regression). */\n  pRegression: number;\n  /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n  pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n  differences: number[],\n  options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n  const n = differences.length;\n  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n  const nResamples = options?.nResamples ?? 1000;\n  const alpha = options?.alpha ?? 0.05;\n\n  const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n  if (n === 1)\n    return {\n      lower: differences[0],\n      upper: differences[0],\n      mean,\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  if (nResamples <= 0)\n    return {\n      lower: round(mean),\n      upper: round(mean),\n      mean: round(mean),\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  // Seeded or random number generator\n  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n  const resampleMeans: number[] = new Array(nResamples);\n  for (let r = 0; r < nResamples; r++) {\n    let sum = 0;\n    for (let i = 0; i < n; i++) {\n      sum += differences[Math.floor(rng() * n)];\n    }\n    resampleMeans[r] = sum / n;\n  }\n\n  resampleMeans.sort((a, b) => a - b);\n\n  const lowerIdx = Math.floor((alpha / 2) * nResamples);\n  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n  // Count resamples below/above zero for directional probabilities\n  // resampleMeans is already sorted, so binary search for zero\n  let belowZero = 0;\n  for (let i = 0; i < nResamples; i++) {\n    if (resampleMeans[i] < 0) belowZero++;\n    else break;\n  }\n  let aboveZero = 0;\n  for (let i = nResamples - 1; i >= 0; i--) {\n    if (resampleMeans[i] > 0) aboveZero++;\n    else break;\n  }\n\n  return {\n    lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n    mean: round(mean),\n    pRegression: round(belowZero / nResamples),\n    pImprovement: round(aboveZero / nResamples),\n  };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n  let state = seed | 0 || 1; // Ensure non-zero\n  return () => {\n    state ^= state << 13;\n    state ^= state >> 17;\n    state ^= state << 5;\n    return (state >>> 0) / 0x100000000;\n  };\n}\n","export function computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\n/**\n * Count, for one scorer across a set of items, how many produced a valid score\n * (`scored`), ran-and-failed (`failed`), or were deliberately skipped by the\n * scorer's `applies` predicate (`skipped`). The single source of truth for the\n * failure-rate trust signal, shared by `runEval`, `rescore`, and `evalCompare`.\n *\n * Three discriminators on the per-scorer `scoreDetails` entry, checked in\n * precedence order so a positive `skipped` marker always wins over the\n * duration heuristic:\n *  - a non-null `scores[name]` ⇒ `scored`;\n *  - else `skipped === true` (the `applies` predicate returned `false`) ⇒\n *    `skipped` — deliberately NOT run, so it belongs in neither the mean nor\n *    the failure-rate denominator;\n *  - else a `duration` ⇒ `failed` (`scoreItem` stamps it whenever a scorer\n *    actually executes: success, throw, or out-of-range);\n *  - else (no score, no marker, no duration) the scorer was skipped by\n *    cancellation (signal aborted before it started) and belongs in no bucket.\n *\n * So `scored + failed` is the honest \"attempted\" denominator for a failure rate\n * — it excludes both `applies`-skips and cancellations and can be `<` the\n * eligible item count.\n *\n * Items whose WORKFLOW errored (`i.error`) are excluded entirely — the scorers\n * never ran for them, so they're not part of the scorer's sample either way.\n */\nexport function scorerCounts(\n  items: readonly {\n    error?: string;\n    scores: Record<string, number | null>;\n    scoreDetails?: Record<string, { duration?: number; skipped?: boolean }>;\n  }[],\n  name: string,\n): { scored: number; failed: number; skipped: number } {\n  let scored = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const i of items) {\n    if (i.error) continue;\n    if (i.scores[name] != null) scored++;\n    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;\n    else if (i.scoreDetails?.[name]?.duration != null) failed++;\n  }\n  return { scored, failed, skipped };\n}\n\n/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */\nexport type ScorerToleranceVerdict = {\n  /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */\n  attempted: number;\n  /** `failed / attempted`, or `0` when nothing was attempted. */\n  rate: number;\n  /** Over tolerance AND something was actually attempted. */\n  exceeds: boolean;\n  /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */\n  zeroSample: boolean;\n};\n\n/**\n * Single source of truth for the type-aware failure-rate decision, shared by the\n * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare\n * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures\n * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`\n * (`failed / attempted`). `exceeds` is never true when nothing was attempted —\n * callers decide their own policy for `zeroSample` (the runner skips it; the\n * compare gate refuses to certify a zero-sample scorer).\n */\nexport function evaluateScorerTolerance(\n  scored: number,\n  failed: number,\n  type: 'llm' | 'deterministic',\n  limit: number,\n): ScorerToleranceVerdict {\n  const attempted = scored + failed;\n  const rate = attempted === 0 ? 0 : failed / attempted;\n  const exceeds = attempted > 0 && (type === 'deterministic' ? failed > 0 : rate > limit);\n  return { attempted, rate, exceeds, zeroSample: attempted === 0 };\n}\n\n/**\n * Run `task` over `items` with a bounded worker pool, preserving input order.\n *\n * Results are written to a pre-allocated array at the item's original index, so\n * `results[i]` always corresponds to `items[i]` regardless of completion order.\n *\n * The helper does NOT catch — each `task` must own its try/catch. This is how\n * callers get `Promise.allSettled` semantics (one failing task never rejects\n * the whole batch) while still using a simple worker-pool: a task that resolves\n * (even after internally handling an error) lets the pool continue draining.\n *\n * Concurrency is clamped to `[1, items.length]` in exactly one place here, and\n * non-finite / fractional values are coerced (NaN/Infinity/`2.5` → a sane\n * integer). Callers must NOT pre-clamp with `Math.min(concurrency, items.length)`\n * and pass the result in — a user/env-supplied `0` (or `NaN` from a config like\n * `Number(process.env.X)`) would otherwise yield zero workers while items\n * remain, so `Promise.all` resolves immediately leaving the results array full\n * of holes (and the caller crashes on the `undefined` entries).\n */\nexport async function mapWithConcurrency<T, R = void>(\n  items: readonly T[],\n  concurrency: number,\n  task: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results = new Array<R>(items.length);\n  // Coerce non-finite (NaN/Infinity) to the floor of 1, and floor fractional\n  // values, BEFORE clamping — `Math.max(1, NaN)` is `NaN`, which would make\n  // `Array.from({ length: NaN })` empty and silently run zero workers.\n  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;\n  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);\n\n  let next = 0;\n  async function runNext(): Promise<void> {\n    while (next < items.length) {\n      const i = next++;\n      results[i] = await task(items[i], i);\n    }\n  }\n\n  await Promise.all(Array.from({ length: workerCount }, () => runNext()));\n  return results;\n}\n","import type {\n  EvalResult,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\nimport { scorerCounts, evaluateScorerTolerance, round } from './utils.js';\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n  scorerName: string,\n  options: EvalCompareOptions | undefined,\n  metadata: Record<string, unknown>,\n): number {\n  // Explicit global threshold\n  if (typeof options?.thresholds === 'number') return options.thresholds;\n\n  // Per-scorer threshold map\n  if (options?.thresholds && typeof options.thresholds === 'object') {\n    const perScorer = options.thresholds[scorerName];\n    if (perScorer != null) return perScorer;\n    // Fall through to auto-calibration if scorer not in map\n  }\n\n  // Auto-calibrate from scorerTypes metadata\n  const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n  if (scorerTypes && scorerName in scorerTypes) {\n    return scorerTypes[scorerName] === 'llm'\n      ? DEFAULT_LLM_THRESHOLD\n      : DEFAULT_DETERMINISTIC_THRESHOLD;\n  }\n\n  // Legacy fallback for results without scorerTypes\n  return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n  baseline: EvalResult | EvalResult[],\n  candidate: EvalResult | EvalResult[],\n  options?: EvalCompareOptions,\n): EvalComparison {\n  const baselineRunsRaw = Array.isArray(baseline) ? baseline : [baseline];\n  const candidateRunsRaw = Array.isArray(candidate) ? candidate : [candidate];\n\n  if (baselineRunsRaw.length === 0 || candidateRunsRaw.length === 0) {\n    throw new Error('Cannot compare empty eval result arrays');\n  }\n\n  // Truncate both sides to a common run count BEFORE computing anything. The\n  // paired bootstrap CI was already restricted to `min(baseline, candidate)`\n  // runs (a CI over paired diffs requires equal-length sides), but means /\n  // regressions / timing / cost previously used the full-length pool on each\n  // side. That hybrid produced an internally inconsistent view: a 5-run\n  // baseline vs 2-run candidate showed `delta = mean(5) - mean(2)` next to\n  // a CI computed over only 2 paired diffs, so the user couldn't tell the\n  // numbers were drawn from different samples. Truncating symmetrically\n  // here keeps every per-scorer figure aligned with the CI's sample size,\n  // and the `runCount` field on each side surfaces the truncation\n  // explicitly so the UI can render \"n=2 of 5 pooled\".\n  const runCount = Math.min(baselineRunsRaw.length, candidateRunsRaw.length);\n  const baselineRuns = baselineRunsRaw.slice(0, runCount);\n  const candidateRuns = candidateRunsRaw.slice(0, runCount);\n\n  // Use the first run as representative for metadata, dataset, scorers.\n  // `partial` detection downstream still walks the FULL pool so\n  // pre-truncation history (`metadata.batchAttempted` from any pooled run)\n  // is honored.\n  const baselineRef = baselineRunsRaw[0];\n  const candidateRef = candidateRunsRaw[0];\n\n  if (baselineRef.dataset !== candidateRef.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n  // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n  const pairedDiffs: Record<string, number[]> = {};\n  for (const name of baselineScorerNames) {\n    pairedDiffs[name] = [];\n  }\n  for (let r = 0; r < runCount; r++) {\n    const bRun = baselineRuns[r];\n    const cRun = candidateRuns[r];\n    const minLength = Math.min(bRun.items.length, cRun.items.length);\n    for (let i = 0; i < minLength; i++) {\n      const bItem = bRun.items[i];\n      const cItem = cRun.items[i];\n      if (bItem.error || cItem.error) continue;\n      for (const name of baselineScorerNames) {\n        const bScore = bItem.scores[name];\n        const cScore = cItem.scores[name];\n        if (bScore != null && cScore != null) {\n          pairedDiffs[name].push(cScore - bScore);\n        }\n      }\n    }\n  }\n\n  // Compute aggregate means across the truncated runs (matches the CI's sample).\n  const baselineMeans: Record<string, number> = {};\n  const candidateMeans: Record<string, number> = {};\n  for (const name of baselineScorerNames) {\n    baselineMeans[name] =\n      baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      baselineRuns.length;\n    candidateMeans[name] =\n      candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      candidateRuns.length;\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMeanRaw = baselineMeans[name];\n    const cMeanRaw = candidateMeans[name];\n    const deltaRaw = cMeanRaw - bMeanRaw;\n    const bMean = round(bMeanRaw);\n    const cMean = round(cMeanRaw);\n    const delta = round(deltaRaw);\n    const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n    const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n    // Per-side scored/failed over the SAME truncated pool the means/CI use, so\n    // a gate (`--max-scorer-error-rate`) reads a sample consistent with the\n    // displayed numbers. Sum across this side's truncated runs (the\n    // `scorerCounts` helper excludes workflow-errored items and counts a\n    // ran-and-failed scorer via its `scoreDetails.duration` discriminator).\n    let baselineScored = 0;\n    let baselineFailed = 0;\n    let baselineSkipped = 0;\n    for (const run of baselineRuns) {\n      const c = scorerCounts(run.items, name);\n      baselineScored += c.scored;\n      baselineFailed += c.failed;\n      baselineSkipped += c.skipped;\n    }\n    let candidateScored = 0;\n    let candidateFailed = 0;\n    let candidateSkipped = 0;\n    for (const run of candidateRuns) {\n      const c = scorerCounts(run.items, name);\n      candidateScored += c.scored;\n      candidateFailed += c.failed;\n      candidateSkipped += c.skipped;\n    }\n\n    const entry: EvalComparison['scorers'][string] = {\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      deltaPercent,\n      baselineScored,\n      baselineFailed,\n      candidateScored,\n      candidateFailed,\n      baselineSkipped,\n      candidateSkipped,\n    };\n\n    // `n` is the PAIRED sample size — items scored on BOTH sides, the only\n    // items a difference can be computed for. Always set it (even 0/1), because\n    // that's exactly the asymmetric case the labeling needs to surface: the\n    // displayed `delta` is built from each side's INDEPENDENT mean (over that\n    // side's own scored items), while the CI below is paired. When `n` is small\n    // relative to `{baseline,candidate}Scored`, the two rest on different\n    // samples. The CI/significance machinery still requires >= 2 paired diffs.\n    const diffs = pairedDiffs[name];\n    entry.n = diffs.length;\n    if (diffs.length >= 2) {\n      const ci = pairedBootstrapCI(diffs);\n      entry.ci = { lower: ci.lower, upper: ci.upper };\n      entry.pRegression = ci.pRegression;\n      entry.pImprovement = ci.pImprovement;\n      // Significant when CI excludes zero AND delta exceeds practical threshold\n      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n    }\n\n    scorers[name] = entry;\n  }\n\n  // Per-item regressions/improvements.\n  // For multi-run: average each item's score across runs to get a stable per-item comparison.\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const itemCount = Math.min(\n    ...baselineRuns.map((r) => r.items.length),\n    ...candidateRuns.map((r) => r.items.length),\n  );\n\n  for (let i = 0; i < itemCount; i++) {\n    // Check if any run has an error for this item\n    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n    if (hasBaselineError || hasCandidateError) continue;\n\n    for (const name of baselineScorerNames) {\n      // Average this item's score across all runs on each side\n      const bScores = baselineRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      const cScores = candidateRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      if (bScores.length === 0 || cScores.length === 0) continue;\n\n      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n      const delta = round(cAvg - bAvg);\n      const threshold = resolveThreshold(name, options, baselineRef.metadata);\n      if (delta < -threshold)\n        regressions.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n      else if (delta > threshold)\n        improvements.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (average across all runs for multi-run)\n  let timing: EvalComparison['timing'];\n  const baselineTimings = baselineRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  const candidateTimings = candidateRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n  }\n\n  // Cost comparison (per-run average for multi-run)\n  let cost: EvalComparison['cost'];\n  const baselineAvgCost =\n    baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n  const candidateAvgCost =\n    candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n  if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n    const deltaRaw = candidateAvgCost - baselineAvgCost;\n    const delta = round(deltaRaw);\n    const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n    cost = {\n      baselineTotal: round(baselineAvgCost),\n      candidateTotal: round(candidateAvgCost),\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      let sigLabel = '';\n      if (s.significant === true) sigLabel = ' (significant)';\n      else if (s.significant === false) sigLabel = ' (not significant)';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  // Run partial detection against the FULL pre-truncation pool. The user's\n  // original selection (e.g. 5 of an attempted 10 runs) determines partial\n  // status; the truncation we did above is a stats-consistency concern, not\n  // a partial-batch one.\n  const baselinePartial = detectPartial(baselineRunsRaw);\n  const candidatePartial = detectPartial(candidateRunsRaw);\n\n  return {\n    baseline: {\n      id: baselineRef.id,\n      metadata: baselineRef.metadata,\n      runCount,\n      ...(baselinePartial ? { partial: baselinePartial } : {}),\n    },\n    candidate: {\n      id: candidateRef.id,\n      metadata: candidateRef.metadata,\n      runCount,\n      ...(candidatePartial ? { partial: candidatePartial } : {}),\n    },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n\n/**\n * Detect whether one side of a comparison was pooled from a partial batch\n * (or a user-selected subset). Returns `{ completed, attempted }` when the\n * pooled run count is less than the original batch's planned run count\n * (read from `metadata.batchAttempted` on any run in the pool); `undefined`\n * otherwise.\n *\n * The signal is the same regardless of cause (batch failed mid-way vs user\n * cherry-picked from the picker) — both result in a smaller-N pool that\n * the consumer should mark visually so the comparison isn't mistaken for\n * an apples-to-apples run.\n *\n * Walks every run looking for the first finite `batchAttempted` rather than\n * trusting `runs[0]` alone — when a user cherry-picks runs from the compare\n * picker, the array order is the user's order, so the first item may be a\n * legacy single-run with no batch metadata while a sibling carries it.\n *\n * If `runs.length > attempted` (more results than were planned), treats the\n * data as suspicious and returns `undefined` — silently marking it \"complete\"\n * would mask data corruption.\n */\nfunction detectPartial(runs: EvalResult[]): { completed: number; attempted: number } | undefined {\n  if (runs.length === 0) return undefined;\n  let attempted: number | undefined;\n  for (const r of runs) {\n    const v = r.metadata?.batchAttempted;\n    if (typeof v === 'number' && Number.isFinite(v)) {\n      attempted = v;\n      break;\n    }\n  }\n  if (attempted === undefined) return undefined;\n  if (runs.length >= attempted) return undefined;\n  return { completed: runs.length, attempted };\n}\n\n/**\n * Decide whether a comparison should be REFUSED under a gate-side scorer\n * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and\n * testable — returns a human-readable refusal reason, or `null` to allow.\n *\n * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY\n * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a\n * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`\n * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the\n * permissive tier) is deliberate: classifying an old artifact's scorers as\n * deterministic would hard-fail them on the first flake, contradicting the\n * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical\n * across sides (enforced by `evalCompare`), so reading types from the baseline\n * is sufficient.\n *\n * Also refuses a scorer that produced ZERO scored items on a side — a mean over\n * an empty sample can't be certified. Counts come from the same truncated pool\n * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).\n */\nexport function evaluateScorerErrorRateGate(\n  comparison: EvalComparison,\n  maxScorerErrorRate: number,\n): string | null {\n  const scorerTypes = (comparison.baseline.metadata?.scorerTypes ?? {}) as Record<string, string>;\n  for (const name of Object.keys(comparison.scorers)) {\n    const s = comparison.scorers[name];\n    const type = scorerTypes[name] === 'deterministic' ? 'deterministic' : 'llm';\n    const limit = type === 'deterministic' ? 0 : maxScorerErrorRate;\n    for (const side of ['baseline', 'candidate'] as const) {\n      const scored = (side === 'baseline' ? s.baselineScored : s.candidateScored) ?? 0;\n      const failed = (side === 'baseline' ? s.baselineFailed : s.candidateFailed) ?? 0;\n      const skipped = (side === 'baseline' ? s.baselineSkipped : s.candidateSkipped) ?? 0;\n      const verdict = evaluateScorerTolerance(scored, failed, type, limit);\n      if (verdict.zeroSample) {\n        // attempted === 0 means the scorer produced no valid scores AND recorded\n        // no ran-and-failed attempts on this side, so a mean over an empty sample\n        // can't be certified. Distinguish the likely cause: a fully `applies`-\n        // skipped scorer (N/A on every item — a scope/config issue, not a fault)\n        // vs. items that errored in the workflow.\n        const cause =\n          skipped > 0\n            ? `every one of its items was N/A (the \\`applies\\` predicate skipped all ${skipped})`\n            : `its items errored or produced no score`;\n        return `scorer \"${name}\" has no scored items on ${side} (${cause}) — cannot certify a mean over an empty sample.`;\n      }\n      if (verdict.exceeds) {\n        const limitStr =\n          type === 'deterministic'\n            ? 'deterministic zero-tolerance'\n            : `${(limit * 100).toFixed(1)}%`;\n        return `scorer \"${name}\" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;\n      }\n    }\n  }\n  return null;\n}\n","import type { Provider } from '@axlsdk/axl';\n\n/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/**\n * Cost contract for thrown scorer errors. A scorer (e.g. `llmScorer`) that\n * incurs provider cost *before* failing attaches that cost to the thrown error\n * via {@link attachScorerErrorCost} so the eval runner still bills it. The\n * cost rides as an enumerable `cost` property (a small, stable public contract\n * custom scorers can also use). These helpers replace scattered `(err as any).cost`\n * reads/writes with one typed, finite-guarded surface.\n */\nexport function attachScorerErrorCost(err: unknown, cost: number | undefined): void {\n  if (\n    err &&\n    typeof err === 'object' &&\n    typeof cost === 'number' &&\n    Number.isFinite(cost) &&\n    cost >= 0\n  ) {\n    (err as { cost?: number }).cost = cost;\n  }\n}\n\n/** Read a non-negative finite cost off a thrown scorer error, or `undefined`. */\nexport function extractScorerErrorCost(err: unknown): number | undefined {\n  const v = err && typeof err === 'object' ? (err as { cost?: unknown }).cost : undefined;\n  return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => { provider: Provider; model: string };\n  /** Abort signal forwarded from the eval runner; scorers should pass it into provider calls. */\n  signal?: AbortSignal;\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\n/**\n * Applicability predicate for a conditional scorer. Returns `true` if this\n * scorer should run for the given item, `false` to skip it.\n *\n * The eval runner runs every scorer against every item, so a scorer that only\n * applies to a subset (a refusal judge for refusal-expected items, a constraint\n * judge for constrained items, …) declares its scope here. A skipped item is\n * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND\n * from the failure-rate denominator, so a conditional scorer stays honest to\n * both. The predicate runs BEFORE the scorer body, so an inapplicable\n * `llmScorer` never makes its provider call.\n *\n * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability\n * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but\n * `output` is available too (e.g. \"only score outputs that refused\"). A\n * predicate that THROWS is a bug, not a skip — it's recorded as a scorer\n * failure, never silently swallowed.\n *\n * The contract is `=> boolean`: return `true` to run, `false` to skip. The\n * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy\n * value (a predicate that forgot to `return`) skips rather than crashes — but\n * authors should return a real boolean. When deriving from a truthy value,\n * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not\n * `=> a.constraints?.length` (which is `number | undefined`, a type error). A\n * predicate that accidentally never returns therefore skips EVERY item — but\n * that surfaces conspicuously as a non-zero `skipped` count (the Studio \"N/A\"\n * chip), not as a silently green mean.\n */\nexport type ScorerApplies<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => boolean;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the scorer is skipped for that item (counted as neither\n   *  scored nor failed). Omit to run on every item (the default).\n   *\n   *  Declared with **method syntax** (not a function-valued property) for parity\n   *  with {@link Scorer.applies}, where the syntax is load-bearing: see the note\n   *  there and `__tests__/scorer-assignability.test-d.ts`. */\n  applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n  /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}.\n   *\n   *  Declared with **method syntax** (not a function-valued `ScorerApplies`\n   *  property) so its parameters are checked bivariantly, exactly like\n   *  {@link score}. `EvalConfig.scorers` erases the generics to\n   *  `Scorer<unknown, unknown, unknown>[]`; under `strictFunctionTypes` a\n   *  property would make those params contravariant, so a concretely-typed\n   *  `Scorer<…, TInput, TAnnotations>` would no longer be assignable to that\n   *  array (the 0.18.1 regression). `__tests__/scorer-assignability.test-d.ts`\n   *  locks this in and is compiled by the `typecheck` gate. */\n  applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n    applies: config.applies,\n  };\n}\n","import type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult, extractScorerErrorCost } from './scorer.js';\nimport type { EvalItem } from './types.js';\nimport { round, mapWithConcurrency } from './utils.js';\n\n/** The subset of an `EvalItem` that `scoreItem` reads and mutates. */\ntype ScorableItem = Pick<\n  EvalItem,\n  'input' | 'output' | 'annotations' | 'scores' | 'scoreDetails' | 'scorerErrors' | 'scorerCost'\n>;\n\n/**\n * Score one item's `output` with every scorer in `scorers`, running up to\n * `scorerConcurrency` of them at once. Mutates `item.scores`,\n * `item.scoreDetails`, `item.scorerErrors`, and `item.scorerCost` in place, and\n * returns the per-item scorer cost so the caller can fold it into its own\n * running total (`totalCost += await scoreItem(...)`).\n *\n * This is the single source of truth for the scoring inner loop, shared by\n * `runEval` and `rescore` — both previously carried a byte-identical copy, so a\n * determinism or cancellation fix had to be made twice.\n *\n * Determinism (independent of scorer completion order):\n *  - `scores`/`scoreDetails` keys are pre-seeded in `scorers` order, so JSON key\n *    order is stable and a scorer skipped by cancellation is deterministically\n *    `null` rather than absent.\n *  - `scorerErrors` are collected name-keyed and flattened in `scorers` order.\n *\n * Cancellation: a scorer not yet started when `signal` aborts is skipped; an\n * in-flight scorer whose call rejects with an `AbortError` is treated as\n * cancellation (leaves its pre-seeded `null`, records no error) — narrowed to\n * the error identity so a genuine bug in another scorer that happens to throw\n * while a sibling triggered the abort is still reported, not swallowed.\n */\nexport async function scoreItem(\n  item: ScorableItem,\n  scorers: readonly Scorer<unknown, unknown, unknown>[],\n  scorerConcurrency: number,\n  scorerContext: ScorerContext,\n  signal?: AbortSignal,\n): Promise<number> {\n  const scoreDetails = (item.scoreDetails ??= {});\n  for (const s of scorers) {\n    item.scores[s.name] = null;\n    scoreDetails[s.name] = { score: null };\n  }\n\n  let itemScorerCost = 0;\n  const scorerErrorsByName: Record<string, string> = {};\n\n  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer) => {\n    if (signal?.aborted) return;\n    const scorerStart = Date.now();\n    try {\n      // Applicability gate: a conditional scorer declares which items it applies\n      // to via `applies`. A `false` verdict skips it entirely — no provider call\n      // for an llmScorer — and records it as NEITHER scored nor failed via the\n      // positive `skipped` marker (no `duration`, so the duration heuristic in\n      // `scorerCounts` can't misread it as a failure). A predicate that THROWS is\n      // a bug, not a skip: it falls through to the catch below and is recorded as\n      // a scorer failure (with a duration), never silently swallowed.\n      if (scorer.applies && !scorer.applies(item.output, item.input, item.annotations)) {\n        item.scores[scorer.name] = null;\n        scoreDetails[scorer.name] = { score: null, skipped: true };\n        return;\n      }\n\n      const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n      const scorerResult = normalizeScorerResult(raw);\n\n      if (scorerResult.cost != null) {\n        itemScorerCost += scorerResult.cost;\n      }\n\n      const scorerDuration = Date.now() - scorerStart;\n\n      if (\n        !Number.isFinite(scorerResult.score) ||\n        scorerResult.score < 0 ||\n        scorerResult.score > 1\n      ) {\n        scorerErrorsByName[scorer.name] =\n          `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;\n        item.scores[scorer.name] = null;\n        scoreDetails[scorer.name] = {\n          score: null,\n          metadata: scorerResult.metadata,\n          duration: scorerDuration,\n          cost: scorerResult.cost,\n        };\n      } else {\n        item.scores[scorer.name] = round(scorerResult.score);\n        scoreDetails[scorer.name] = {\n          score: round(scorerResult.score),\n          metadata: scorerResult.metadata,\n          duration: scorerDuration,\n          cost: scorerResult.cost,\n        };\n      }\n    } catch (err) {\n      // A cancelled call (AbortError) is not a scoring failure — leave the\n      // pre-seeded null and record nothing.\n      if ((err as { name?: string })?.name === 'AbortError') return;\n      const errCost = extractScorerErrorCost(err);\n      if (errCost != null) itemScorerCost += errCost;\n      scorerErrorsByName[scorer.name] =\n        `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`;\n      item.scores[scorer.name] = null;\n      scoreDetails[scorer.name] = {\n        score: null,\n        duration: Date.now() - scorerStart,\n        cost: errCost,\n      };\n    }\n  });\n\n  const orderedErrors = scorers\n    .map((s) => scorerErrorsByName[s.name])\n    .filter((e): e is string => e != null);\n  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;\n  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n  return itemScorerCost;\n}\n","import type { AxlRuntime, CallTiming } from '@axlsdk/axl';\nimport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  ItemModelTiming,\n  ModelTimingStats,\n  RunEvalOptions,\n} from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport type { DegradedScorer } from './types.js';\nimport {\n  computeStats,\n  mapWithConcurrency,\n  scorerCounts,\n  evaluateScorerTolerance,\n} from './utils.js';\nimport { scoreItem } from './score-item.js';\nimport { randomUUID } from 'node:crypto';\n\n/** The `modelTiming` rollup `AxlRuntime.trackExecution` returns — sums plus the\n *  raw `samples` array. Derived from the runtime's own signature so a removed\n *  field is a type error here; `absorbTiming` copies fields explicitly so an\n *  ADDED core field cannot silently widen the persisted `item.timing`. */\ntype TrackedModelTiming = NonNullable<\n  Awaited<ReturnType<AxlRuntime['trackExecution']>>['modelTiming']\n>;\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/**\n * Extract a user-returned cost only if it's a non-negative finite number.\n * Guards against workflows that return `{ cost: 'free' }`, `{ cost: NaN }`,\n * `{ cost: -1 }`, `{ cost: Infinity }`, etc. — the TS type says `cost?: number`\n * but at runtime we can't trust that.\n *\n * Negative costs are rejected because (a) cost is a USD amount and negative\n * values are nonsensical, and (b) a negative cost would silently shrink\n * `totalCost` below the budget limit check at the processItem level,\n * letting a buggy/malicious workflow run unbounded. `0` is preserved\n * because free operations on paid models are valid.\n *\n * When a user-supplied value is rejected (present but invalid), we log\n * once per item via `console.warn` so the type violation is visible\n * rather than silently swallowed.\n */\nfunction extractUserCost(result: unknown, label = 'executeWorkflow'): number | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const raw = (result as { cost?: unknown }).cost;\n  if (raw === undefined) return undefined;\n  if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return raw;\n\n  console.warn(\n    `[axl-eval] Ignoring invalid \\`cost\\` from ${label} return: expected non-negative finite number, got ${typeof raw === 'number' ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`,\n  );\n  return undefined;\n}\n\n/**\n * Extract a user-returned metadata record only if it's a plain object.\n * Rejects arrays, null, scalars, and exotic object types (Date, Map, Set,\n * Error, class instances) that would satisfy a loose `typeof === 'object'`\n * check but break `Record<string, unknown>` assumptions in downstream\n * consumers (spread, Object.entries, property access).\n */\nfunction extractUserMetadata(\n  result: unknown,\n  label = 'executeWorkflow',\n): Record<string, unknown> | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const meta = (result as { metadata?: unknown }).metadata;\n  if (meta === undefined) return undefined;\n  if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${meta === null ? 'null' : Array.isArray(meta) ? 'array' : typeof meta}.`,\n    );\n    return undefined;\n  }\n  // Reject exotic objects (Date, Map, Set, Error, class instances) whose\n  // prototype chain differs from Object.prototype. Those pass `typeof ===\n  // 'object'` but don't behave like `Record<string, unknown>`.\n  const proto = Object.getPrototypeOf(meta);\n  if (proto !== Object.prototype && proto !== null) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${(proto?.constructor?.name as string | undefined) ?? 'exotic object'}.`,\n    );\n    return undefined;\n  }\n  return meta as Record<string, unknown>;\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n  runtime: AxlRuntime,\n  options?: RunEvalOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  // Snapshot dataset-load diagnostics (e.g. annotation keys the schema dropped)\n  // so we can surface them on EvalResult.metadata for any consumer — mirrors the\n  // console.warn the dataset already emits. Read synchronously after getItems()\n  // (no await between); defensive against hand-rolled Dataset objects that don't\n  // expose the field.\n  const droppedAnnotationKeys = [...(config.dataset.droppedAnnotationKeys ?? [])];\n  const concurrency = config.concurrency ?? 5;\n  // Per-item scorer fan-out. Defaults to the same value as item `concurrency`\n  // (parallel-by-default), so the worst-case concurrent judge calls is\n  // `concurrency × scorerConcurrency`. The provider layer backs off on\n  // 429/503/529; users who need a tighter ceiling lower item `concurrency`.\n  const scorerConcurrency = config.scorerConcurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n    signal: options?.signal,\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  // Run-level per-call latency samples, keyed by effective model URI. Filled\n  // synchronously from each item's `trackExecution` result and never persisted:\n  // `summary.modelTiming` needs real per-call values to produce a real\n  // distribution, while `item.timing` keeps only the compact sums. Items run\n  // concurrently, but each append is a synchronous push, so no interleaving is\n  // possible; sample ORDER across items is arbitrary and irrelevant to stats.\n  const runCallSamples = new Map<string, CallTiming[]>();\n  /**\n   * Split a `trackExecution` timing rollup into the compact per-item sums that\n   * get persisted and the raw per-call blocks that feed `summary.modelTiming`.\n   * Both branches of the item loop go through here so the two paths can't drift.\n   */\n  function absorbTiming(rollup: TrackedModelTiming | undefined, evalItem: EvalItem): void {\n    if (!rollup) return;\n    const perItem: Record<string, ItemModelTiming> = {};\n    for (const [model, bucket] of Object.entries(rollup)) {\n      // Explicit field copy, not a rest spread: a rest spread would carry any\n      // future core-side field straight into every persisted eval artifact.\n      const sums: ItemModelTiming = {\n        calls: bucket.calls,\n        queuedMs: bucket.queuedMs,\n        retryMs: bucket.retryMs,\n        wireMs: bucket.wireMs,\n        ...(bucket.firstTokenMs !== undefined && bucket.firstTokenCalls !== undefined\n          ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls }\n          : {}),\n      };\n      perItem[model] = sums;\n      // `samples` is opt-in on the core side and both call sites below request\n      // it, so it is never actually absent here — but it is typed optional and\n      // is treated that way rather than asserted, so a future caller that omits\n      // the flag degrades to \"no distribution\" instead of throwing.\n      const samples = bucket.samples;\n      if (!samples) continue;\n      // Appended one at a time rather than by spread: a long-running item can\n      // hold thousands of calls, and `push(...arr)` passes them as arguments.\n      const pooled = runCallSamples.get(model) ?? [];\n      for (const sample of samples) pooled.push(sample);\n      runCallSamples.set(model, pooled);\n    }\n    evalItem.timing = perItem;\n  }\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (options?.signal?.aborted) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Cancelled',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      // When captureTraces is on, wrap the user's executeWorkflow in\n      // trackExecution so we get a per-item TraceEvent[] scoped via\n      // AsyncLocalStorage. trackExecution's scope walks the parent chain, so\n      // any nested trackExecution (e.g. from the CLI) continues to work.\n      if (options?.captureTraces) {\n        const tracked = await runtime.trackExecution(\n          async () => executeWorkflow(item.input, runtime),\n          { captureTraces: true, captureTimingSamples: true },\n        );\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = tracked.result.output;\n        // Prefer user-returned cost/metadata; fall back to tracked values.\n        // Type-guarded so a non-number cost (e.g. `{ cost: 'free' }`) or a\n        // non-object metadata (e.g. an array or scalar) doesn't silently\n        // corrupt downstream math or shape expectations.\n        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;\n        if (tracked.unpriced) evalItem.unpriced = true;\n        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;\n        absorbTiming(tracked.modelTiming, evalItem);\n        if (tracked.traces && tracked.traces.length > 0) {\n          evalItem.traces = tracked.traces;\n        }\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      } else {\n        // Default path. We wrap in trackExecution ONLY to read the per-model\n        // timing rollup — nothing else about this branch changes. `cost`,\n        // `unpriced` and `metadata` deliberately keep coming from the user's\n        // return value alone, exactly as before: falling back to tracked cost\n        // here would newly populate `metadata.models`, and would let a plain\n        // run with a `budget` abort mid-run. That is its own decision.\n        //\n        // Guarded like the `resolveProvider` check above, because a hand-rolled\n        // or duck-typed runtime (the suite's `{} as AxlRuntime`) has no\n        // `trackExecution`; such a runtime simply reports no timing.\n        //\n        // Cost: one extra `trace` listener per in-flight item (so at most\n        // `concurrency` of them, not one per item in the dataset). Each listener\n        // begins with an O(1) `Set` miss on the event's `executionId` and\n        // returns immediately for anything outside its own scope, so a streaming\n        // item emitting thousands of `token` events pays a hash lookup per\n        // event, not a walk. The `captureTraces` path already ran this way at\n        // the same concurrency.\n        let trackedTiming: TrackedModelTiming | undefined;\n        let result: Awaited<ReturnType<typeof executeWorkflow>>;\n        if (typeof runtime.trackExecution === 'function') {\n          // `captureTimingSamples` is what makes `summary.modelTiming` a real\n          // per-call distribution; the sums alone could only yield a mean.\n          const tracked = await runtime.trackExecution(\n            async () => executeWorkflow(item.input, runtime),\n            { captureTraces: false, captureTimingSamples: true },\n          );\n          result = tracked.result;\n          trackedTiming = tracked.modelTiming;\n        } else {\n          result = await executeWorkflow(item.input, runtime);\n        }\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = result.output;\n        evalItem.cost = extractUserCost(result);\n        const meta = extractUserMetadata(result);\n        if (meta) evalItem.metadata = meta;\n        absorbTiming(trackedTiming, evalItem);\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      // Failed items are exactly when per-item traces are most valuable —\n      // recover them from the captured-traces side-channel `trackExecution`\n      // attaches to the error on the failure path.\n      if (options?.captureTraces) {\n        const captured = (err as { axlCapturedTraces?: unknown }).axlCapturedTraces;\n        if (Array.isArray(captured) && captured.length > 0) {\n          evalItem.traces = captured as EvalItem['traces'];\n        }\n      }\n      evalItems[itemIndex] = evalItem;\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    // Score the item's output with all scorers (bounded fan-out). scoreItem\n    // owns the determinism + cancellation + cost logic shared with rescore();\n    // it returns the per-item scorer cost to fold into the running total.\n    // NB: keep the await on its own line — `totalCost += await …` would read\n    // totalCost BEFORE suspending, losing updates across concurrent items.\n    const itemScorerCost = await scoreItem(\n      evalItem,\n      config.scorers,\n      scorerConcurrency,\n      scorerContext,\n      options?.signal,\n    );\n    totalCost += itemScorerCost;\n    evalItems[itemIndex] = evalItem;\n    options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n  }\n\n  // processItem writes into the pre-allocated `evalItems` closure array (the\n  // source of truth) and returns void — the pool's returned array is ignored.\n  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    const { scored, failed, skipped } = scorerCounts(evalItems, name);\n    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of config.scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  // Failure-rate trust signal (opt-in). When `failOnScorerErrorRate` is set, a\n  // scorer is degraded when it's deterministic and failed at all (a failure is\n  // a bug, not noise), OR it's an LLM judge whose failure rate over its\n  // ATTEMPTED items exceeds tolerance. We never throw — the result is still\n  // useful; we flag it and let the CLI/consumer decide the exit code. Invalid\n  // limits are ignored loudly so a typo can't silently disable the gate.\n  let degraded: DegradedScorer[] | undefined;\n  const rawLimit = config.failOnScorerErrorRate;\n  if (rawLimit != null) {\n    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {\n      console.warn(\n        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`,\n      );\n    } else {\n      for (const name of scorerNames) {\n        const stats = scorerStats[name];\n        const scored = stats.scored ?? 0;\n        const failed = stats.failed ?? 0;\n        const type = scorerTypes[name] === 'llm' ? 'llm' : 'deterministic';\n        const limit = type === 'deterministic' ? 0 : rawLimit;\n        const verdict = evaluateScorerTolerance(scored, failed, type, limit);\n        // A scorer that never ran (zeroSample) isn't degraded at the source — the\n        // run produced no basis to judge it. (The compare gate treats zero-sample\n        // differently: it refuses to certify it.)\n        if (verdict.exceeds) {\n          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });\n        }\n      }\n    }\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  // Per-model provider latency: one distribution per field, every sample a real\n  // provider call. A ten-call item contributes ten samples, so `wireMs.mean` is\n  // the exact call-weighted mean and `wireMs.p95` is a genuine call percentile.\n  //\n  // This is why the raw `CallTiming` blocks are pooled during the run instead of\n  // being re-derived from `item.timing`: per-item sums can yield a mean, but no\n  // percentile that describes calls rather than items.\n  //\n  // `firstTokenMs` runs over the streaming calls only — a non-streaming call is\n  // excluded from the sample rather than entered as `0`, which would report a\n  // first-token latency no call achieved.\n  const modelTiming: Record<string, ModelTimingStats> | undefined =\n    runCallSamples.size > 0\n      ? Object.fromEntries(\n          [...runCallSamples.entries()].map(([model, samples]) => {\n            const firstToken = samples\n              .map((s) => s.firstTokenMs)\n              .filter((v): v is number => v != null);\n            const stats: ModelTimingStats = {\n              calls: samples.length,\n              wireMs: computeStats(samples.map((s) => s.wireMs)),\n              queuedMs: computeStats(samples.map((s) => s.queuedMs)),\n              retryMs: computeStats(samples.map((s) => s.retryMs)),\n              ...(firstToken.length > 0\n                ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length }\n                : {}),\n            };\n            return [model, stats];\n          }),\n        )\n      : undefined;\n\n  // Aggregate per-model LLM call counts across all items\n  const totalModelCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.modelCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n      const itemModels = item.metadata?.models;\n      if (Array.isArray(itemModels)) {\n        for (const m of itemModels) {\n          if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // models: unique list sorted by total calls (most-called first)\n  // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n  const modelsMeta: Record<string, unknown> = {};\n  if (totalModelCalls.size > 0) {\n    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n    modelsMeta.models = sorted.map(([m]) => m);\n    modelsMeta.modelCounts = Object.fromEntries(sorted);\n  }\n\n  // Aggregate per-workflow call counts across all items (parallel to models).\n  // Workflows come from trackExecution's trace-event collection — callers\n  // don't specify workflow names anywhere, they just appear because the\n  // runtime emits workflow_start events for every execute() call.\n  const totalWorkflowCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.workflowCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique workflows per item\n      const itemWorkflows = item.metadata?.workflows;\n      if (Array.isArray(itemWorkflows)) {\n        for (const w of itemWorkflows) {\n          if (typeof w === 'string')\n            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // Fall back to config.workflow when the callback bypassed the runtime's\n  // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n  // metadata.workflows array non-empty for the common test-harness case.\n  const workflowsMeta: Record<string, unknown> = {};\n  if (totalWorkflowCalls.size > 0) {\n    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n    workflowsMeta.workflows = sorted.map(([w]) => w);\n    workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n  } else if (config.workflow) {\n    workflowsMeta.workflows = [config.workflow];\n    workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n  }\n\n  options?.onProgress?.({ type: 'run_done', totalItems: items.length, failures });\n\n  return {\n    id,\n    dataset: config.dataset.name,\n    metadata: {\n      ...config.metadata,\n      scorerTypes,\n      ...modelsMeta,\n      ...workflowsMeta,\n      ...(droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}),\n    },\n    timestamp: new Date().toISOString(),\n    totalCost,\n    ...(evalItems.some((item) => item.unpriced) ? { unpriced: true } : {}),\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: {\n      count: items.length,\n      failures,\n      scorers: scorerStats,\n      timing,\n      ...(modelTiming ? { modelTiming } : {}),\n      ...(degraded ? { degraded } : {}),\n    },\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { computeStats, mapWithConcurrency, scorerCounts } from './utils.js';\nimport { scoreItem } from './score-item.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n  /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */\n  concurrency?: number;\n  /** Per-item scorer fan-out (how many scorers run concurrently within one item).\n   *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent\n   *  judge calls is `concurrency × scorerConcurrency`. */\n  scorerConcurrency?: number;\n  /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can\n   *  be cancelled mid-flight. Also checked between items to short-circuit\n   *  remaining work. Mirrors `RunEvalOptions.signal`. */\n  signal?: AbortSignal;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n  result: EvalResult,\n  scorers: Scorer[],\n  runtime: AxlRuntime,\n  options?: RescoreOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const concurrency = options?.concurrency ?? 5;\n  const scorerConcurrency = options?.scorerConcurrency ?? 5;\n\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n    signal: options?.signal,\n  };\n\n  const rescored: EvalItem[] = new Array(result.items.length);\n  let totalCost = 0;\n\n  async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n    // Short-circuit if the rescore has been cancelled — matches runEval's\n    // between-items signal check (runner.ts) so cancellation behaves the same\n    // in both paths. In-flight LLM scorer calls additionally abort via\n    // scorerContext.signal → provider.chat({ signal }).\n    if (options?.signal?.aborted) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: 'cancelled',\n        scores: {},\n      };\n      return;\n    }\n\n    // Pass through error items without scoring\n    if (original.error) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: original.error,\n        scores: {},\n      };\n      return;\n    }\n\n    // Preserve per-item traces from the original run: rescore re-runs scorers\n    // but leaves the workflow untouched, so the original execution traces\n    // remain accurate and useful for diagnosing score changes.\n    const item: EvalItem = {\n      input: original.input,\n      annotations: original.annotations,\n      output: original.output,\n      metadata: original.metadata,\n      traces: original.traces,\n      scores: {},\n      scoreDetails: {},\n    };\n\n    // Same shared scoring path as runEval — determinism, cancellation, and cost\n    // accounting all live in scoreItem. Keep the await off the `+=` line so the\n    // read-modify-write isn't split by suspension across concurrent items.\n    const itemScorerCost = await scoreItem(\n      item,\n      scorers,\n      scorerConcurrency,\n      scorerContext,\n      options?.signal,\n    );\n    totalCost += itemScorerCost;\n    rescored[itemIndex] = item;\n  }\n\n  // rescoreItem writes into the pre-allocated `rescored` closure array (the\n  // source of truth) and returns void — the pool's returned array is ignored.\n  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));\n\n  const failures = rescored.filter((i) => i.error).length;\n  const scorerNames = scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = rescored\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    // Same scored/failed surfacing as runEval (table + Studio parity). NOTE:\n    // rescore deliberately does NOT support `failOnScorerErrorRate` — it takes\n    // RescoreOptions, not EvalConfig, so there's no degradation gate here. The\n    // counts are informational; gating belongs to the run that produced output.\n    const { scored, failed, skipped } = scorerCounts(rescored, name);\n    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  return {\n    id: randomUUID(),\n    dataset: result.dataset,\n    metadata: (() => {\n      // Strip run group membership — rescored results are independent evaluations.\n      // metadata.workflows is preserved via ...rest so the rescored result keeps\n      // the same workflow attribution as the original.\n      //\n      // Also strip the scorer-filtered stamp: whether THIS rescore ran a subset\n      // is a property of the rescore invocation, not the source run. A full\n      // rescore of a `--scorers`-filtered run would otherwise inherit a stale\n      // `scorerFiltered: true` + a `scorersRun` listing scorers it didn't even\n      // use — falsely tripping the compare gate and the Studio banner.\n      const rest: Record<string, unknown> = { ...result.metadata };\n      delete rest.runGroupId;\n      delete rest.runIndex;\n      delete rest.scorerFiltered;\n      delete rest.scorersRun;\n      const merged: Record<string, unknown> = {\n        ...rest,\n        rescored: true,\n        originalId: result.id,\n        scorerTypes,\n      };\n      // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n      // as a top-level string field with no `metadata.workflows`. Migrate it\n      // forward so rescored results from old artifacts retain their workflow\n      // attribution under the modern shape.\n      if (!Array.isArray(merged.workflows)) {\n        const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n        if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n          merged.workflows = [legacyWorkflow];\n          merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n        }\n      }\n      return merged;\n    })(),\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: rescored,\n    summary: { count: result.items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n  runGroupId: string;\n  runCount: number;\n  /**\n   * Workflow names observed across all runs in the group, deduped and ordered\n   * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n   * multi-run group produced by `axl-eval --runs N` typically has one entry;\n   * heterogeneous groups (custom callbacks) can have multiple.\n   */\n  workflows: string[];\n  dataset: string;\n  totalCost: number;\n  totalDuration: number;\n  scorers: Record<\n    string,\n    {\n      mean: number;\n      std: number;\n      min: number;\n      max: number;\n      /** Total valid scores summed across all runs in the group. */\n      scored?: number;\n      /** Total ran-and-failed scorer invocations summed across all runs. */\n      failed?: number;\n      /** Total `applies`-skipped (N/A) items summed across all runs. Excluded\n       *  from both the mean and the failure-rate denominator. */\n      skipped?: number;\n    }\n  >;\n  timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n  if (values.length <= 1) return 0;\n  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n  return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n  if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n  const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n  const { dataset } = runs[0];\n\n  // Union workflow names across all runs in the group, first-seen first.\n  // Most groups have one, but custom callbacks can produce heterogeneous ones.\n  const seenWorkflows = new Set<string>();\n  const workflows: string[] = [];\n  for (const run of runs) {\n    const list = run.metadata.workflows;\n    if (Array.isArray(list)) {\n      for (const w of list) {\n        if (typeof w === 'string' && !seenWorkflows.has(w)) {\n          seenWorkflows.add(w);\n          workflows.push(w);\n        }\n      }\n    }\n  }\n\n  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n  // Collect per-scorer means from each run\n  const scorerNames = Object.keys(runs[0].summary.scorers);\n  const scorers: MultiRunSummary['scorers'] = {};\n\n  for (const name of scorerNames) {\n    // A run where this scorer scored ZERO items has an empty-sample mean of 0\n    // (computeStats([]) → 0), which would drag the aggregate mean-of-means toward\n    // 0 as if it were a real 0.0 score. Exclude such runs from the mean/min/max/std\n    // — but only when we can tell: `scored` is absent on pre-0.18.0 artifacts, so\n    // those runs are kept (old behavior). If EVERY run scored nothing, fall back to\n    // all runs so we never divide by zero.\n    const contributing = runs.filter((r) => {\n      const sc = r.summary.scorers[name]?.scored;\n      return sc == null || sc > 0;\n    });\n    const meanSource = contributing.length > 0 ? contributing : runs;\n    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);\n    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n    const minMean = Math.min(...means);\n    const maxMean = Math.max(...means);\n\n    // Sum the per-run sample sizes across ALL runs (they describe the whole\n    // group's thinning, independent of which runs contributed to the mean).\n    // Read with `?? 0` — old artifacts predate these fields.\n    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);\n    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);\n    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);\n\n    scorers[name] = {\n      mean: round(meanOfMeans),\n      std: round(std(means, meanOfMeans)),\n      min: round(minMean),\n      max: round(maxMean),\n      scored,\n      failed,\n      skipped,\n    };\n  }\n\n  // Timing aggregation\n  let timing: MultiRunSummary['timing'];\n  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n  if (timingMeans.length === runs.length && timingMeans.length > 0) {\n    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n    timing = {\n      mean: round(meanTiming),\n      std: round(std(timingMeans, meanTiming)),\n    };\n  }\n\n  return {\n    runGroupId,\n    runCount: runs.length,\n    workflows,\n    dataset,\n    totalCost: round(totalCost),\n    totalDuration: round(totalDuration),\n    scorers,\n    timing,\n  };\n}\n"],"mappings":";AAmBO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAO,MAAM,IAAI;AAAA,MACjB,OAAO,MAAM,IAAI;AAAA,MACjB,MAAM,MAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAO,MAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAM,MAAM,IAAI;AAAA,IAChB,aAAa,MAAM,YAAY,UAAU;AAAA,IACzC,cAAc,MAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;ACrGO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEO,SAASA,OAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AA2BO,SAAS,aACd,OAKA,MACqD;AACrD,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,KAAK,OAAO;AACrB,QAAI,EAAE,MAAO;AACb,QAAI,EAAE,OAAO,IAAI,KAAK,KAAM;AAAA,aACnB,EAAE,eAAe,IAAI,GAAG,YAAY,KAAM;AAAA,aAC1C,EAAE,eAAe,IAAI,GAAG,YAAY,KAAM;AAAA,EACrD;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;AAuBO,SAAS,wBACd,QACA,QACA,MACA,OACwB;AACxB,QAAM,YAAY,SAAS;AAC3B,QAAM,OAAO,cAAc,IAAI,IAAI,SAAS;AAC5C,QAAM,UAAU,YAAY,MAAM,SAAS,kBAAkB,SAAS,IAAI,OAAO;AACjF,SAAO,EAAE,WAAW,MAAM,SAAS,YAAY,cAAc,EAAE;AACjE;AAqBA,eAAsB,mBACpB,OACA,aACA,MACc;AACd,QAAM,UAAU,IAAI,MAAS,MAAM,MAAM;AAIzC,QAAM,YAAY,OAAO,SAAS,WAAW,IAAI,KAAK,MAAM,WAAW,IAAI;AAC3E,QAAM,cAAc,MAAM,WAAW,IAAI,IAAI,KAAK,IAAI,KAAK,IAAI,GAAG,SAAS,GAAG,MAAM,MAAM;AAE1F,MAAI,OAAO;AACX,iBAAe,UAAyB;AACtC,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,IAAI;AACV,cAAQ,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,GAAG,CAAC;AAAA,IACrC;AAAA,EACF;AAEA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,YAAY,GAAG,MAAM,QAAQ,CAAC,CAAC;AACtE,SAAO;AACT;;;AClIA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,kBAAkB,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACtE,QAAM,mBAAmB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAE1E,MAAI,gBAAgB,WAAW,KAAK,iBAAiB,WAAW,GAAG;AACjE,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAaA,QAAM,WAAW,KAAK,IAAI,gBAAgB,QAAQ,iBAAiB,MAAM;AACzE,QAAM,eAAe,gBAAgB,MAAM,GAAG,QAAQ;AACtD,QAAM,gBAAgB,iBAAiB,MAAM,GAAG,QAAQ;AAMxD,QAAM,cAAc,gBAAgB,CAAC;AACrC,QAAM,eAAe,iBAAiB,CAAC;AAEvC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQC,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAOtE,QAAI,iBAAiB;AACrB,QAAI,iBAAiB;AACrB,QAAI,kBAAkB;AACtB,eAAW,OAAO,cAAc;AAC9B,YAAM,IAAI,aAAa,IAAI,OAAO,IAAI;AACtC,wBAAkB,EAAE;AACpB,wBAAkB,EAAE;AACpB,yBAAmB,EAAE;AAAA,IACvB;AACA,QAAI,kBAAkB;AACtB,QAAI,kBAAkB;AACtB,QAAI,mBAAmB;AACvB,eAAW,OAAO,eAAe;AAC/B,YAAM,IAAI,aAAa,IAAI,OAAO,IAAI;AACtC,yBAAmB,EAAE;AACrB,yBAAmB,EAAE;AACrB,0BAAoB,EAAE;AAAA,IACxB;AAEA,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AASA,UAAM,QAAQ,YAAY,IAAI;AAC9B,UAAM,IAAI,MAAM;AAChB,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AAExB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAM/J,QAAM,kBAAkB,cAAc,eAAe;AACrD,QAAM,mBAAmB,cAAc,gBAAgB;AAEvD,SAAO;AAAA,IACL,UAAU;AAAA,MACR,IAAI,YAAY;AAAA,MAChB,UAAU,YAAY;AAAA,MACtB;AAAA,MACA,GAAI,kBAAkB,EAAE,SAAS,gBAAgB,IAAI,CAAC;AAAA,IACxD;AAAA,IACA,WAAW;AAAA,MACT,IAAI,aAAa;AAAA,MACjB,UAAU,aAAa;AAAA,MACvB;AAAA,MACA,GAAI,mBAAmB,EAAE,SAAS,iBAAiB,IAAI,CAAC;AAAA,IAC1D;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;AAuBA,SAAS,cAAc,MAA0E;AAC/F,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,MAAI;AACJ,aAAW,KAAK,MAAM;AACpB,UAAM,IAAI,EAAE,UAAU;AACtB,QAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,GAAG;AAC/C,kBAAY;AACZ;AAAA,IACF;AAAA,EACF;AACA,MAAI,cAAc,OAAW,QAAO;AACpC,MAAI,KAAK,UAAU,UAAW,QAAO;AACrC,SAAO,EAAE,WAAW,KAAK,QAAQ,UAAU;AAC7C;AAqBO,SAAS,4BACd,YACA,oBACe;AACf,QAAM,cAAe,WAAW,SAAS,UAAU,eAAe,CAAC;AACnE,aAAW,QAAQ,OAAO,KAAK,WAAW,OAAO,GAAG;AAClD,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,YAAY,IAAI,MAAM,kBAAkB,kBAAkB;AACvE,UAAM,QAAQ,SAAS,kBAAkB,IAAI;AAC7C,eAAW,QAAQ,CAAC,YAAY,WAAW,GAAY;AACrD,YAAM,UAAU,SAAS,aAAa,EAAE,iBAAiB,EAAE,oBAAoB;AAC/E,YAAM,UAAU,SAAS,aAAa,EAAE,iBAAiB,EAAE,oBAAoB;AAC/E,YAAM,WAAW,SAAS,aAAa,EAAE,kBAAkB,EAAE,qBAAqB;AAClF,YAAM,UAAU,wBAAwB,QAAQ,QAAQ,MAAM,KAAK;AACnE,UAAI,QAAQ,YAAY;AAMtB,cAAM,QACJ,UAAU,IACN,yEAAyE,OAAO,MAChF;AACN,eAAO,WAAW,IAAI,4BAA4B,IAAI,KAAK,KAAK;AAAA,MAClE;AACA,UAAI,QAAQ,SAAS;AACnB,cAAM,WACJ,SAAS,kBACL,iCACA,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AACjC,eAAO,WAAW,IAAI,aAAa,QAAQ,OAAO,KAAK,QAAQ,CAAC,CAAC,QAAQ,IAAI,KAAK,MAAM,IAAI,QAAQ,SAAS,eAAe,QAAQ;AAAA,MACtI;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;;;AC5ZO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AAUO,SAAS,sBAAsB,KAAc,MAAgC;AAClF,MACE,OACA,OAAO,QAAQ,YACf,OAAO,SAAS,YAChB,OAAO,SAAS,IAAI,KACpB,QAAQ,GACR;AACA,IAAC,IAA0B,OAAO;AAAA,EACpC;AACF;AAGO,SAAS,uBAAuB,KAAkC;AACvE,QAAM,IAAI,OAAO,OAAO,QAAQ,WAAY,IAA2B,OAAO;AAC9E,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,KAAK,KAAK,IAAI,IAAI;AACrE;AAuFO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,IACd,SAAS,OAAO;AAAA,EAClB;AACF;;;ACrGA,eAAsB,UACpB,MACA,SACA,mBACA,eACA,QACiB;AACjB,QAAM,eAAgB,KAAK,iBAAiB,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,SAAK,OAAO,EAAE,IAAI,IAAI;AACtB,iBAAa,EAAE,IAAI,IAAI,EAAE,OAAO,KAAK;AAAA,EACvC;AAEA,MAAI,iBAAiB;AACrB,QAAM,qBAA6C,CAAC;AAEpD,QAAM,mBAAmB,SAAS,mBAAmB,OAAOC,YAAW;AACrE,QAAI,QAAQ,QAAS;AACrB,UAAM,cAAc,KAAK,IAAI;AAC7B,QAAI;AAQF,UAAIA,QAAO,WAAW,CAACA,QAAO,QAAQ,KAAK,QAAQ,KAAK,OAAO,KAAK,WAAW,GAAG;AAChF,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,qBAAaA,QAAO,IAAI,IAAI,EAAE,OAAO,MAAM,SAAS,KAAK;AACzD;AAAA,MACF;AAEA,YAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,YAAM,eAAe,sBAAsB,GAAG;AAE9C,UAAI,aAAa,QAAQ,MAAM;AAC7B,0BAAkB,aAAa;AAAA,MACjC;AAEA,YAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,UACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,2BAAmBA,QAAO,IAAI,IAC5B,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AACnH,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,qBAAaA,QAAO,IAAI,IAAI;AAAA,UAC1B,OAAO;AAAA,UACP,UAAU,aAAa;AAAA,UACvB,UAAU;AAAA,UACV,MAAM,aAAa;AAAA,QACrB;AAAA,MACF,OAAO;AACL,aAAK,OAAOA,QAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACnD,qBAAaD,QAAO,IAAI,IAAI;AAAA,UAC1B,OAAOC,OAAM,aAAa,KAAK;AAAA,UAC/B,UAAU,aAAa;AAAA,UACvB,UAAU;AAAA,UACV,MAAM,aAAa;AAAA,QACrB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AAGZ,UAAK,KAA2B,SAAS,aAAc;AACvD,YAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAI,WAAW,KAAM,mBAAkB;AACvC,yBAAmBD,QAAO,IAAI,IAC5B,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AACpF,WAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,mBAAaA,QAAO,IAAI,IAAI;AAAA,QAC1B,OAAO;AAAA,QACP,UAAU,KAAK,IAAI,IAAI;AAAA,QACvB,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF,CAAC;AAED,QAAM,gBAAgB,QACnB,IAAI,CAAC,MAAM,mBAAmB,EAAE,IAAI,CAAC,EACrC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,MAAI,cAAc,SAAS,EAAG,MAAK,eAAe;AAClD,OAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,SAAO;AACT;;;ACvGA,SAAS,kBAAkB;AAU3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAkBA,SAAS,gBAAgB,QAAiB,QAAQ,mBAAuC;AACvF,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,MAAO,OAA8B;AAC3C,MAAI,QAAQ,OAAW,QAAO;AAC9B,MAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,KAAK,OAAO,EAAG,QAAO;AAExE,UAAQ;AAAA,IACN,6CAA6C,KAAK,qDAAqD,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI,OAAO,GAAG;AAAA,EAC3J;AACA,SAAO;AACT;AASA,SAAS,oBACP,QACA,QAAQ,mBAC6B;AACrC,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,OAAQ,OAAkC;AAChD,MAAI,SAAS,OAAW,QAAO;AAC/B,MAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;AACpE,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAuC,SAAS,OAAO,SAAS,MAAM,QAAQ,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,IACnK;AACA,WAAO;AAAA,EACT;AAIA,QAAM,QAAQ,OAAO,eAAe,IAAI;AACxC,MAAI,UAAU,OAAO,aAAa,UAAU,MAAM;AAChD,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAwC,OAAO,aAAa,QAA+B,eAAe;AAAA,IAClK;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAsB,QACpB,QACA,iBAIA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAM5C,QAAM,wBAAwB,CAAC,GAAI,OAAO,QAAQ,yBAAyB,CAAC,CAAE;AAC9E,QAAM,cAAc,OAAO,eAAe;AAK1C,QAAM,oBAAoB,OAAO,qBAAqB;AACtD,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,IACA,QAAQ,SAAS;AAAA,EACnB;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAQrB,QAAM,iBAAiB,oBAAI,IAA0B;AAMrD,WAAS,aAAa,QAAwC,UAA0B;AACtF,QAAI,CAAC,OAAQ;AACb,UAAM,UAA2C,CAAC;AAClD,eAAW,CAAC,OAAO,MAAM,KAAK,OAAO,QAAQ,MAAM,GAAG;AAGpD,YAAM,OAAwB;AAAA,QAC5B,OAAO,OAAO;AAAA,QACd,UAAU,OAAO;AAAA,QACjB,SAAS,OAAO;AAAA,QAChB,QAAQ,OAAO;AAAA,QACf,GAAI,OAAO,iBAAiB,UAAa,OAAO,oBAAoB,SAChE,EAAE,cAAc,OAAO,cAAc,iBAAiB,OAAO,gBAAgB,IAC7E,CAAC;AAAA,MACP;AACA,cAAQ,KAAK,IAAI;AAKjB,YAAM,UAAU,OAAO;AACvB,UAAI,CAAC,QAAS;AAGd,YAAM,SAAS,eAAe,IAAI,KAAK,KAAK,CAAC;AAC7C,iBAAW,UAAU,QAAS,QAAO,KAAK,MAAM;AAChD,qBAAe,IAAI,OAAO,MAAM;AAAA,IAClC;AACA,aAAS,SAAS;AAAA,EACpB;AAEA,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,SAAS,QAAQ,SAAS;AAC5B,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAKF,UAAI,SAAS,eAAe;AAC1B,cAAM,UAAU,MAAM,QAAQ;AAAA,UAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,UAC/C,EAAE,eAAe,MAAM,sBAAsB,KAAK;AAAA,QACpD;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,QAAQ,OAAO;AAKjC,iBAAS,OAAO,gBAAgB,QAAQ,MAAM,KAAK,QAAQ;AAC3D,YAAI,QAAQ,SAAU,UAAS,WAAW;AAC1C,iBAAS,WAAW,oBAAoB,QAAQ,MAAM,KAAK,QAAQ;AACnE,qBAAa,QAAQ,aAAa,QAAQ;AAC1C,YAAI,QAAQ,UAAU,QAAQ,OAAO,SAAS,GAAG;AAC/C,mBAAS,SAAS,QAAQ;AAAA,QAC5B;AACA,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF,OAAO;AAmBL,YAAI;AACJ,YAAI;AACJ,YAAI,OAAO,QAAQ,mBAAmB,YAAY;AAGhD,gBAAM,UAAU,MAAM,QAAQ;AAAA,YAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,YAC/C,EAAE,eAAe,OAAO,sBAAsB,KAAK;AAAA,UACrD;AACA,mBAAS,QAAQ;AACjB,0BAAgB,QAAQ;AAAA,QAC1B,OAAO;AACL,mBAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AAAA,QACpD;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,OAAO;AACzB,iBAAS,OAAO,gBAAgB,MAAM;AACtC,cAAM,OAAO,oBAAoB,MAAM;AACvC,YAAI,KAAM,UAAS,WAAW;AAC9B,qBAAa,eAAe,QAAQ;AACpC,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAIhE,UAAI,SAAS,eAAe;AAC1B,cAAM,WAAY,IAAwC;AAC1D,YAAI,MAAM,QAAQ,QAAQ,KAAK,SAAS,SAAS,GAAG;AAClD,mBAAS,SAAS;AAAA,QACpB;AAAA,MACF;AACA,gBAAU,SAAS,IAAI;AACvB,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAOA,UAAM,iBAAiB,MAAM;AAAA,MAC3B;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA,SAAS;AAAA,IACX;AACA,iBAAa;AACb,cAAU,SAAS,IAAI;AACvB,aAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAAA,EAClF;AAIA,QAAM,mBAAmB,OAAO,aAAa,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC;AAE9E,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,UAAM,EAAE,QAAQ,QAAQ,QAAQ,IAAI,aAAa,WAAW,IAAI;AAChE,gBAAY,IAAI,IAAI,EAAE,GAAG,aAAa,MAAM,GAAG,QAAQ,QAAQ,QAAQ;AAAA,EACzE;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAQA,MAAI;AACJ,QAAM,WAAW,OAAO;AACxB,MAAI,YAAY,MAAM;AACpB,QAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,KAAK,WAAW,GAAG;AAC9D,cAAQ;AAAA,QACN,sDAAsD,QAAQ;AAAA,MAChE;AAAA,IACF,OAAO;AACL,iBAAW,QAAQ,aAAa;AAC9B,cAAM,QAAQ,YAAY,IAAI;AAC9B,cAAM,SAAS,MAAM,UAAU;AAC/B,cAAM,SAAS,MAAM,UAAU;AAC/B,cAAM,OAAO,YAAY,IAAI,MAAM,QAAQ,QAAQ;AACnD,cAAM,QAAQ,SAAS,kBAAkB,IAAI;AAC7C,cAAM,UAAU,wBAAwB,QAAQ,QAAQ,MAAM,KAAK;AAInE,YAAI,QAAQ,SAAS;AACnB,WAAC,aAAa,CAAC,GAAG,KAAK,EAAE,QAAQ,MAAM,MAAM,QAAQ,MAAM,OAAO,MAAM,QAAQ,OAAO,CAAC;AAAA,QAC1F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAahE,QAAM,cACJ,eAAe,OAAO,IAClB,OAAO;AAAA,IACL,CAAC,GAAG,eAAe,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,OAAO,MAAM;AACtD,YAAM,aAAa,QAChB,IAAI,CAAC,MAAM,EAAE,YAAY,EACzB,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,QAA0B;AAAA,QAC9B,OAAO,QAAQ;AAAA,QACf,QAAQ,aAAa,QAAQ,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAAA,QACjD,UAAU,aAAa,QAAQ,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC;AAAA,QACrD,SAAS,aAAa,QAAQ,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;AAAA,QACnD,GAAI,WAAW,SAAS,IACpB,EAAE,cAAc,aAAa,UAAU,GAAG,iBAAiB,WAAW,OAAO,IAC7E,CAAC;AAAA,MACP;AACA,aAAO,CAAC,OAAO,KAAK;AAAA,IACtB,CAAC;AAAA,EACH,IACA;AAGN,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,WAAS,aAAa,EAAE,MAAM,YAAY,YAAY,MAAM,QAAQ,SAAS,CAAC;AAE9E,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,MACH,GAAI,sBAAsB,SAAS,IAAI,EAAE,sBAAsB,IAAI,CAAC;AAAA,IACtE;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,GAAI,UAAU,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,EAAE,UAAU,KAAK,IAAI,CAAC;AAAA,IACpE,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS;AAAA,MACP,OAAO,MAAM;AAAA,MACb;AAAA,MACA,SAAS;AAAA,MACT;AAAA,MACA,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,MACrC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,EACF;AACF;;;AC/eA,SAAS,cAAAE,mBAAkB;AAmB3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,oBAAoB,SAAS,qBAAqB;AAExD,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,IACA,QAAQ,SAAS;AAAA,EACnB;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAK/E,QAAI,SAAS,QAAQ,SAAS;AAC5B,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAGA,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAKA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,SAAS;AAAA,MACjB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAKA,UAAM,iBAAiB,MAAM;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,IACX;AACA,iBAAa;AACb,aAAS,SAAS,IAAI;AAAA,EACxB;AAIA,QAAM,mBAAmB,OAAO,OAAO,aAAa,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC;AAErF,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AAKtC,UAAM,EAAE,QAAQ,QAAQ,QAAQ,IAAI,aAAa,UAAU,IAAI;AAC/D,gBAAY,IAAI,IAAI,EAAE,GAAG,aAAa,MAAM,GAAG,QAAQ,QAAQ,QAAQ;AAAA,EACzE;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,IAAIA,YAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAUf,YAAM,OAAgC,EAAE,GAAG,OAAO,SAAS;AAC3D,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;AC3KA,SAAS,cAAAC,mBAAkB;AAmC3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,cAAyBC,YAAW;AACzE,QAAM,EAAE,QAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAO9B,UAAM,eAAe,KAAK,OAAO,CAAC,MAAM;AACtC,YAAM,KAAK,EAAE,QAAQ,QAAQ,IAAI,GAAG;AACpC,aAAO,MAAM,QAAQ,KAAK;AAAA,IAC5B,CAAC;AACD,UAAM,aAAa,aAAa,SAAS,IAAI,eAAe;AAC5D,UAAM,QAAQ,WAAW,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AACtE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAKjC,UAAM,SAAS,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,UAAU,IAAI,CAAC;AACtF,UAAM,SAAS,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,UAAU,IAAI,CAAC;AACtF,UAAM,UAAU,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,WAAW,IAAI,CAAC;AAExF,YAAQ,IAAI,IAAI;AAAA,MACd,MAAMC,OAAM,WAAW;AAAA,MACvB,KAAKA,OAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAKA,OAAM,OAAO;AAAA,MAClB,KAAKA,OAAM,OAAO;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAMA,OAAM,UAAU;AAAA,MACtB,KAAKA,OAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA,WAAWA,OAAM,SAAS;AAAA,IAC1B,eAAeA,OAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["round","round","scorer","round","randomUUID","randomUUID","randomUUID","round"]}

package/dist/cli.cjs

@@ -531,6 +531,27 @@ async function runEval(config, executeWorkflow, runtime, options) {
   const evalItems = new Array(items.length);
   let totalCost = 0;
   let budgetExceeded = false;
+  const runCallSamples = /* @__PURE__ */ new Map();
+  function absorbTiming(rollup, evalItem) {
+    if (!rollup) return;
+    const perItem = {};
+    for (const [model, bucket] of Object.entries(rollup)) {
+      const sums = {
+        calls: bucket.calls,
+        queuedMs: bucket.queuedMs,
+        retryMs: bucket.retryMs,
+        wireMs: bucket.wireMs,
+        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
+      };
+      perItem[model] = sums;
+      const samples = bucket.samples;
+      if (!samples) continue;
+      const pooled = runCallSamples.get(model) ?? [];
+      for (const sample of samples) pooled.push(sample);
+      runCallSamples.set(model, pooled);
+    }
+    evalItem.timing = perItem;
+  }
   async function processItem(item, itemIndex) {
     if (options?.signal?.aborted) {
       evalItems[itemIndex] = {
@@ -565,13 +586,14 @@ async function runEval(config, executeWorkflow, runtime, options) {
       if (options?.captureTraces) {
         const tracked = await runtime.trackExecution(
           async () => executeWorkflow(item.input, runtime),
-          { captureTraces: true }
+          { captureTraces: true, captureTimingSamples: true }
         );
         evalItem.duration = Date.now() - itemStart;
         evalItem.output = tracked.result.output;
         evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
         if (tracked.unpriced) evalItem.unpriced = true;
         evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        absorbTiming(tracked.modelTiming, evalItem);
         if (tracked.traces && tracked.traces.length > 0) {
           evalItem.traces = tracked.traces;
         }
@@ -579,12 +601,24 @@ async function runEval(config, executeWorkflow, runtime, options) {
           totalCost += evalItem.cost;
         }
       } else {
-        const result = await executeWorkflow(item.input, runtime);
+        let trackedTiming;
+        let result;
+        if (typeof runtime.trackExecution === "function") {
+          const tracked = await runtime.trackExecution(
+            async () => executeWorkflow(item.input, runtime),
+            { captureTraces: false, captureTimingSamples: true }
+          );
+          result = tracked.result;
+          trackedTiming = tracked.modelTiming;
+        } else {
+          result = await executeWorkflow(item.input, runtime);
+        }
         evalItem.duration = Date.now() - itemStart;
         evalItem.output = result.output;
         evalItem.cost = extractUserCost(result);
         const meta = extractUserMetadata(result);
         if (meta) evalItem.metadata = meta;
+        absorbTiming(trackedTiming, evalItem);
         if (evalItem.cost != null) {
           totalCost += evalItem.cost;
         }
@@ -652,6 +686,19 @@ async function runEval(config, executeWorkflow, runtime, options) {
   }
   const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
   const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const modelTiming = runCallSamples.size > 0 ? Object.fromEntries(
+    [...runCallSamples.entries()].map(([model, samples]) => {
+      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
+      const stats = {
+        calls: samples.length,
+        wireMs: computeStats(samples.map((s) => s.wireMs)),
+        queuedMs: computeStats(samples.map((s) => s.queuedMs)),
+        retryMs: computeStats(samples.map((s) => s.retryMs)),
+        ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
+      };
+      return [model, stats];
+    })
+  ) : void 0;
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -723,6 +770,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
       failures,
       scorers: scorerStats,
       timing,
+      ...modelTiming ? { modelTiming } : {},
       ...degraded ? { degraded } : {}
     }
   };
@@ -1042,6 +1090,24 @@ function parseEvalArgs(args) {
   };
 }
 
+// src/cli-format.ts
+function formatModelTimingLines(modelTiming, nameWidth) {
+  if (!modelTiming) return [];
+  return Object.entries(modelTiming).map(([model, t]) => {
+    const parts = [`wire ${formatMs(t.wireMs.mean)}/${formatMs(t.wireMs.p95)}`];
+    if (t.firstTokenMs) {
+      parts.push(`first token ${formatMs(t.firstTokenMs.mean)}/${formatMs(t.firstTokenMs.p95)}`);
+    }
+    parts.push(`queued ${formatMs(t.queuedMs.mean)}`);
+    parts.push(`retries ${formatMs(t.retryMs.mean)}`);
+    const calls = `${t.calls} call${t.calls === 1 ? "" : "s"}`;
+    return `    ${model.padEnd(Math.max(nameWidth - 2, 0))}  ${parts.join(" \xB7 ")}  (${calls}, mean/p95 per call)`;
+  });
+}
+function formatMs(ms) {
+  return `${Math.round(ms)}ms`;
+}
+
 // src/cli.ts
 var import_meta = {};
 function refuseToGate(reason) {
@@ -1403,6 +1469,7 @@ function formatTable(result) {
       `  ${"Timing".padEnd(maxNameLen)}  ${(t.mean / 1e3).toFixed(2).padStart(colWidth)}s ${(t.p50 / 1e3).toFixed(2).padStart(colWidth)}s ${(t.p95 / 1e3).toFixed(2).padStart(colWidth)}s`
     );
   }
+  lines.push(...formatModelTimingLines(result.summary.modelTiming, maxNameLen));
   const durationSec = (result.duration / 1e3).toFixed(1);
   const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : "$0.00";
   lines.push("");