@axlsdk/eval 0.13.6 → 0.13.8

package/dist/cli.cjs

@@ -28,76 +28,223 @@ var import_node_fs2 = require("fs");
 var import_promises = require("fs/promises");
 var path = __toESM(require("path"), 1);
 
-// src/compare.ts
+// src/bootstrap.ts
+function pairedBootstrapCI(differences, options) {
+  const n = differences.length;
+  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };
+  const nResamples = options?.nResamples ?? 1e3;
+  const alpha = options?.alpha ?? 0.05;
+  const mean = differences.reduce((a, b) => a + b, 0) / n;
+  if (n === 1)
+    return {
+      lower: differences[0],
+      upper: differences[0],
+      mean,
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  if (nResamples <= 0)
+    return {
+      lower: round(mean),
+      upper: round(mean),
+      mean: round(mean),
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();
+  const resampleMeans = new Array(nResamples);
+  for (let r = 0; r < nResamples; r++) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      sum += differences[Math.floor(rng() * n)];
+    }
+    resampleMeans[r] = sum / n;
+  }
+  resampleMeans.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * nResamples);
+  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;
+  let belowZero = 0;
+  for (let i = 0; i < nResamples; i++) {
+    if (resampleMeans[i] < 0) belowZero++;
+    else break;
+  }
+  let aboveZero = 0;
+  for (let i = nResamples - 1; i >= 0; i--) {
+    if (resampleMeans[i] > 0) aboveZero++;
+    else break;
+  }
+  return {
+    lower: round(resampleMeans[Math.max(0, lowerIdx)]),
+    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),
+    mean: round(mean),
+    pRegression: round(belowZero / nResamples),
+    pImprovement: round(aboveZero / nResamples)
+  };
+}
 function round(n) {
-  return Math.round(n * 100) / 100;
+  return Math.round(n * 1e4) / 1e4;
+}
+function xorshift32(seed) {
+  let state = seed | 0 || 1;
+  return () => {
+    state ^= state << 13;
+    state ^= state >> 17;
+    state ^= state << 5;
+    return (state >>> 0) / 4294967296;
+  };
 }
-function evalCompare(baseline, candidate) {
-  if (baseline.dataset !== candidate.dataset) {
+
+// src/compare.ts
+function round2(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+var DEFAULT_LLM_THRESHOLD = 0.05;
+var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
+var LEGACY_THRESHOLD = 0.1;
+function resolveThreshold(scorerName, options, metadata) {
+  if (typeof options?.thresholds === "number") return options.thresholds;
+  if (options?.thresholds && typeof options.thresholds === "object") {
+    const perScorer = options.thresholds[scorerName];
+    if (perScorer != null) return perScorer;
+  }
+  const scorerTypes = metadata.scorerTypes;
+  if (scorerTypes && scorerName in scorerTypes) {
+    return scorerTypes[scorerName] === "llm" ? DEFAULT_LLM_THRESHOLD : DEFAULT_DETERMINISTIC_THRESHOLD;
+  }
+  return LEGACY_THRESHOLD;
+}
+function evalCompare(baseline, candidate, options) {
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  if (baselineRuns.length === 0 || candidateRuns.length === 0) {
+    throw new Error("Cannot compare empty eval result arrays");
+  }
+  const baselineRef = baselineRuns[0];
+  const candidateRef = candidateRuns[0];
+  if (baselineRef.dataset !== candidateRef.dataset) {
     throw new Error(
-      `Cannot compare evals from different datasets: "${baseline.dataset}" vs "${candidate.dataset}"`
+      `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
     );
   }
-  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();
-  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();
+  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();
   if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
     throw new Error(`Cannot compare evals with different scorers`);
   }
+  const pairedDiffs = {};
+  for (const name of baselineScorerNames) {
+    pairedDiffs[name] = [];
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    const minLength = Math.min(bRun.items.length, cRun.items.length);
+    for (let i = 0; i < minLength; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      if (bItem.error || cItem.error) continue;
+      for (const name of baselineScorerNames) {
+        const bScore = bItem.scores[name];
+        const cScore = cItem.scores[name];
+        if (bScore != null && cScore != null) {
+          pairedDiffs[name].push(cScore - bScore);
+        }
+      }
+    }
+  }
+  const baselineMeans = {};
+  const candidateMeans = {};
+  for (const name of baselineScorerNames) {
+    baselineMeans[name] = baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / baselineRuns.length;
+    candidateMeans[name] = candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / candidateRuns.length;
+  }
   const scorers = {};
   for (const name of baselineScorerNames) {
-    const bMean = baseline.summary.scorers[name].mean;
-    const cMean = candidate.summary.scorers[name].mean;
-    const delta = round(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
-    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+    const bMeanRaw = baselineMeans[name];
+    const cMeanRaw = candidateMeans[name];
+    const deltaRaw = cMeanRaw - bMeanRaw;
+    const bMean = round2(bMeanRaw);
+    const cMean = round2(cMeanRaw);
+    const delta = round2(deltaRaw);
+    const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
+    const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    const entry = {
+      baselineMean: bMean,
+      candidateMean: cMean,
+      delta,
+      deltaPercent
+    };
+    const diffs = pairedDiffs[name];
+    if (diffs.length >= 2) {
+      const ci = pairedBootstrapCI(diffs);
+      entry.ci = { lower: ci.lower, upper: ci.upper };
+      entry.pRegression = ci.pRegression;
+      entry.pImprovement = ci.pImprovement;
+      entry.n = diffs.length;
+      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
+      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
+    }
+    scorers[name] = entry;
   }
   const regressions = [];
   const improvements = [];
-  const minLength = Math.min(baseline.items.length, candidate.items.length);
-  for (let i = 0; i < minLength; i++) {
-    const bItem = baseline.items[i];
-    const cItem = candidate.items[i];
-    if (bItem.error || cItem.error) continue;
+  const itemCount = Math.min(
+    ...baselineRuns.map((r) => r.items.length),
+    ...candidateRuns.map((r) => r.items.length)
+  );
+  for (let i = 0; i < itemCount; i++) {
+    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);
+    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);
+    if (hasBaselineError || hasCandidateError) continue;
     for (const name of baselineScorerNames) {
-      const bScore = bItem.scores[name];
-      const cScore = cItem.scores[name];
-      if (bScore == null || cScore == null) continue;
-      const delta = round(cScore - bScore);
-      if (delta < -0.1)
+      const bScores = baselineRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      const cScores = candidateRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      if (bScores.length === 0 || cScores.length === 0) continue;
+      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
+      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
+      const delta = round2(cAvg - bAvg);
+      const threshold = resolveThreshold(name, options, baselineRef.metadata);
+      if (delta < -threshold)
         regressions.push({
           itemIndex: i,
-          input: bItem.input,
+          input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
           delta
         });
-      else if (delta > 0.1)
+      else if (delta > threshold)
         improvements.push({
           itemIndex: i,
-          input: bItem.input,
+          input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
           delta
         });
     }
   }
   let timing;
-  if (baseline.summary.timing && candidate.summary.timing) {
-    const bMean = baseline.summary.timing.mean;
-    const cMean = candidate.summary.timing.mean;
-    const delta = round(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
-    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+  const baselineTimings = baselineRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  const candidateTimings = candidateRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (baselineTimings.length > 0 && candidateTimings.length > 0) {
+    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
+    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
+    const delta = round2(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round2((cMean - bMean) / bMean * 100) : 0;
+    timing = { baselineMean: round2(bMean), candidateMean: round2(cMean), delta, deltaPercent };
   }
   let cost;
-  if (baseline.totalCost > 0 || candidate.totalCost > 0) {
-    const delta = round(candidate.totalCost - baseline.totalCost);
-    const deltaPercent = baseline.totalCost > 0 ? round(delta / baseline.totalCost * 100) : 0;
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
+  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
+    const deltaRaw = candidateAvgCost - baselineAvgCost;
+    const delta = round2(deltaRaw);
+    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : 0;
     cost = {
-      baselineTotal: baseline.totalCost,
-      candidateTotal: candidate.totalCost,
+      baselineTotal: round2(baselineAvgCost),
+      candidateTotal: round2(candidateAvgCost),
       delta,
       deltaPercent
     };
@@ -108,8 +255,11 @@ function evalCompare(baseline, candidate) {
     if (Math.abs(s.delta) > 1e-3) {
       const direction = s.delta > 0 ? "improves" : "regresses";
       const sign = s.delta > 0 ? "+" : "";
+      let sigLabel = "";
+      if (s.significant === true) sigLabel = " (significant)";
+      else if (s.significant === false) sigLabel = " (not significant)";
       parts.push(
-        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`
       );
     }
   }
@@ -121,10 +271,10 @@ function evalCompare(baseline, candidate) {
     const dir = cost.delta > 0 ? "more expensive" : "cheaper";
     parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
   }
-  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
   return {
-    baseline: { id: baseline.id, metadata: baseline.metadata },
-    candidate: { id: candidate.id, metadata: candidate.metadata },
+    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },
+    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },
     scorers,
     timing,
     cost,
@@ -139,8 +289,7 @@ function normalizeScorerResult(result) {
   return typeof result === "number" ? { score: result } : result;
 }
 
-// src/runner.ts
-var import_node_crypto = require("crypto");
+// src/utils.ts
 function computeStats(scores) {
   if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
   const sorted = [...scores].sort((a, b) => a - b);
@@ -149,11 +298,14 @@ function computeStats(scores) {
   const max = sorted[sorted.length - 1];
   const p50 = sorted[Math.floor(sorted.length * 0.5)];
   const p95 = sorted[Math.floor(sorted.length * 0.95)];
-  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
+  return { mean: round3(mean), min: round3(min), max: round3(max), p50: round3(p50), p95: round3(p95) };
 }
-function round2(n) {
+function round3(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+
+// src/runner.ts
+var import_node_crypto = require("crypto");
 function parseCost(cost) {
   const match = cost.match(/^\$?([\d.]+)$/);
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
@@ -201,6 +353,7 @@ async function runEval(config, executeWorkflow, runtime) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.output = result.output;
       evalItem.cost = result.cost;
+      if (result.metadata) evalItem.metadata = result.metadata;
       if (result.cost != null) {
         totalCost += result.cost;
       }
@@ -243,9 +396,9 @@ async function runEval(config, executeWorkflow, runtime) {
             cost: scorerResult.cost
           };
         } else {
-          evalItem.scores[scorer.name] = round2(scorerResult.score);
+          evalItem.scores[scorer.name] = round3(scorerResult.score);
           evalItem.scoreDetails[scorer.name] = {
-            score: round2(scorerResult.score),
+            score: round3(scorerResult.score),
             metadata: scorerResult.metadata,
             duration: scorerDuration,
             cost: scorerResult.cost
@@ -290,11 +443,42 @@ async function runEval(config, executeWorkflow, runtime) {
   }
   const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
   const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const scorerTypes = {};
+  for (const s of config.scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  const totalModelCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.modelCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [m, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
+      }
+    } else {
+      const itemModels = item.metadata?.models;
+      if (Array.isArray(itemModels)) {
+        for (const m of itemModels) {
+          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const modelsMeta = {};
+  if (totalModelCalls.size > 0) {
+    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
+    modelsMeta.models = sorted.map(([m]) => m);
+    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  }
   return {
     id,
     workflow: config.workflow,
     dataset: config.dataset.name,
-    metadata: config.metadata ?? {},
+    metadata: {
+      ...config.metadata,
+      scorerTypes,
+      ...modelsMeta
+    },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
     duration: Date.now() - startTime,
@@ -303,6 +487,182 @@ async function runEval(config, executeWorkflow, runtime) {
   };
 }
 
+// src/rescore.ts
+var import_node_crypto2 = require("crypto");
+async function rescore(result, scorers, runtime, options) {
+  const startTime = Date.now();
+  const concurrency = options?.concurrency ?? 5;
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    }
+  };
+  const rescored = new Array(result.items.length);
+  let totalCost = 0;
+  async function rescoreItem(original, itemIndex) {
+    if (original.error) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: original.error,
+        scores: {}
+      };
+      return;
+    }
+    const item = {
+      input: original.input,
+      annotations: original.annotations,
+      output: original.output,
+      metadata: original.metadata,
+      scores: {},
+      scoreDetails: {}
+    };
+    let itemScorerCost = 0;
+    for (const scorer of scorers) {
+      const scorerStart = Date.now();
+      try {
+        const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);
+        const scorerResult = normalizeScorerResult(raw);
+        if (scorerResult.cost != null) {
+          itemScorerCost += scorerResult.cost;
+          totalCost += scorerResult.cost;
+        }
+        const scorerDuration = Date.now() - scorerStart;
+        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+          if (!item.scorerErrors) item.scorerErrors = [];
+          item.scorerErrors.push(
+            `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
+          );
+          item.scores[scorer.name] = null;
+          item.scoreDetails[scorer.name] = {
+            score: null,
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        } else {
+          item.scores[scorer.name] = round3(scorerResult.score);
+          item.scoreDetails[scorer.name] = {
+            score: round3(scorerResult.score),
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        }
+      } catch (err) {
+        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
+        if (errCost != null) {
+          itemScorerCost += errCost;
+          totalCost += errCost;
+        }
+        if (!item.scorerErrors) item.scorerErrors = [];
+        item.scorerErrors.push(
+          `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        item.scores[scorer.name] = null;
+        item.scoreDetails[scorer.name] = {
+          score: null,
+          duration: Date.now() - scorerStart,
+          cost: errCost
+        };
+      }
+    }
+    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    rescored[itemIndex] = item;
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < result.items.length) {
+      const currentIndex = index++;
+      await rescoreItem(result.items[currentIndex], currentIndex);
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, result.items.length) },
+    () => runNext()
+  );
+  await Promise.all(workers);
+  const failures = rescored.filter((i) => i.error).length;
+  const scorerNames = scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  const scorerTypes = {};
+  for (const s of scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  return {
+    id: (0, import_node_crypto2.randomUUID)(),
+    workflow: result.workflow,
+    dataset: result.dataset,
+    metadata: (() => {
+      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      return { ...rest, rescored: true, originalId: result.id, scorerTypes };
+    })(),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: rescored,
+    summary: { count: result.items.length, failures, scorers: scorerStats }
+  };
+}
+
+// src/multi-run.ts
+var import_node_crypto3 = require("crypto");
+function std(values, mean) {
+  if (values.length <= 1) return 0;
+  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);
+  return Math.sqrt(sumSqDiffs / (values.length - 1));
+}
+function aggregateRuns(runs) {
+  if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
+  const runGroupId = runs[0].metadata.runGroupId ?? (0, import_node_crypto3.randomUUID)();
+  const { workflow, dataset } = runs[0];
+  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
+  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+  const scorerNames = Object.keys(runs[0].summary.scorers);
+  const scorers = {};
+  for (const name of scorerNames) {
+    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
+    const minMean = Math.min(...means);
+    const maxMean = Math.max(...means);
+    scorers[name] = {
+      mean: round3(meanOfMeans),
+      std: round3(std(means, meanOfMeans)),
+      min: round3(minMean),
+      max: round3(maxMean)
+    };
+  }
+  let timing;
+  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (timingMeans.length === runs.length && timingMeans.length > 0) {
+    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
+    timing = {
+      mean: round3(meanTiming),
+      std: round3(std(timingMeans, meanTiming))
+    };
+  }
+  return {
+    runGroupId,
+    runCount: runs.length,
+    workflow,
+    dataset,
+    totalCost: round3(totalCost),
+    totalDuration: round3(totalDuration),
+    scorers,
+    timing
+  };
+}
+
 // src/cli-utils.ts
 var import_node_path = require("path");
 var import_node_fs = require("fs");
@@ -367,17 +727,27 @@ async function registerConditions(conditions) {
 
 // src/cli.ts
 var import_meta = {};
-var KNOWN_FLAGS = /* @__PURE__ */ new Set(["--output", "--config", "--conditions", "--fail-on-regression"]);
+var KNOWN_FLAGS = /* @__PURE__ */ new Set([
+  "--output",
+  "--config",
+  "--conditions",
+  "--fail-on-regression",
+  "--threshold",
+  "--runs"
+]);
 async function main() {
   const args = process.argv.slice(2);
   if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
     console.log(`
 Usage:
   axl-eval <path>                         Run eval file(s)
+  axl-eval <path> --runs <n>              Run eval N times (multi-run)
   axl-eval <path> --output <file>         Save results to JSON
   axl-eval <path> --config <file>         Use config file for runtime
   axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval rescore <results> <eval-file>  Re-run scorers on saved outputs
   axl-eval compare <a> <b>                Compare two eval result files
+  axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
 
 Config auto-detection (when --config is not specified):
@@ -393,18 +763,40 @@ When no config is found, a bare AxlRuntime is created (providers from env vars).
     await runCompare(args.slice(1));
     return;
   }
+  if (args[0] === "rescore") {
+    await runRescore(args.slice(1));
+    return;
+  }
   await runEvalCommand(args);
 }
+function parseThresholdArg(args) {
+  const idx = args.indexOf("--threshold");
+  if (idx === -1 || idx + 1 >= args.length) return void 0;
+  const value = args[idx + 1];
+  if (/^[\d.]+$/.test(value)) return parseFloat(value);
+  const map = {};
+  for (const pair of value.split(",")) {
+    const [key, val] = pair.split("=");
+    if (key && val != null) map[key.trim()] = parseFloat(val.trim());
+  }
+  return Object.keys(map).length > 0 ? map : void 0;
+}
 async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
-  const files = args.filter((a) => !a.startsWith("--"));
+  const thresholds = parseThresholdArg(args);
+  const files = args.filter(
+    (a, i) => !a.startsWith("--") && !(i > 0 && args[i - 1] === "--threshold")
+  );
   if (files.length !== 2) {
-    console.error("Usage: axl-eval compare <baseline.json> <candidate.json>");
+    console.error("Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]");
     process.exit(1);
   }
-  const baseline = JSON.parse(await (0, import_promises.readFile)(files[0], "utf-8"));
-  const candidate = JSON.parse(await (0, import_promises.readFile)(files[1], "utf-8"));
-  const comparison = evalCompare(baseline, candidate);
+  const baselineRaw = JSON.parse(await (0, import_promises.readFile)(files[0], "utf-8"));
+  const candidateRaw = JSON.parse(await (0, import_promises.readFile)(files[1], "utf-8"));
+  const baseline = baselineRaw;
+  const candidate = candidateRaw;
+  const compareOptions = thresholds != null ? { thresholds } : void 0;
+  const comparison = evalCompare(baseline, candidate, compareOptions);
   console.log(
     `
 Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})
@@ -412,14 +804,26 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   );
   const scorerNames = Object.keys(comparison.scorers);
   const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);
-  console.log(`  ${"Scorer".padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);
-  console.log(`  ${"".padEnd(maxNameLen, "-")}  --------  ---------  --------  ------`);
+  const hasCI = scorerNames.some((n) => comparison.scorers[n].ci != null);
+  const ciHeader = hasCI ? "  CI 95%            Sig" : "";
+  const ciRule = hasCI ? "  ----------------  ---" : "";
+  console.log(`  ${"Scorer".padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change${ciHeader}`);
+  console.log(`  ${"".padEnd(maxNameLen, "-")}  --------  ---------  --------  ------${ciRule}`);
   for (const name of scorerNames) {
     const s = comparison.scorers[name];
     const sign = s.delta > 0 ? "+" : "";
-    console.log(
-      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + "%").padStart(6)}`
-    );
+    let line = `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + "%").padStart(6)}`;
+    if (hasCI) {
+      if (s.ci) {
+        const lo = (s.ci.lower >= 0 ? "+" : "") + s.ci.lower.toFixed(4);
+        const hi = (s.ci.upper >= 0 ? "+" : "") + s.ci.upper.toFixed(4);
+        const sig = s.significant ? "  *" : "   ";
+        line += `  [${lo}, ${hi}]${sig}`;
+      } else {
+        line += "  \u2014".padEnd(22);
+      }
+    }
+    console.log(line);
   }
   if (comparison.timing) {
     const t = comparison.timing;
@@ -436,9 +840,10 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
       `  Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`
     );
   }
+  const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;
   const stable = Math.max(
     0,
-    baseline.items.length - comparison.regressions.length - comparison.improvements.length
+    baselineRef.items.length - comparison.regressions.length - comparison.improvements.length
   );
   console.log(
     `
@@ -446,8 +851,52 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
 `
   );
   if (failOnRegression && comparison.regressions.length > 0) {
+    const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);
+    if (hasSignificance) {
+      const hasSignificantRegression = scorerNames.some(
+        (n) => comparison.scorers[n].significant === true && comparison.scorers[n].delta < 0
+      );
+      if (hasSignificantRegression) process.exit(1);
+    } else {
+      process.exit(1);
+    }
+  }
+}
+async function runRescore(args) {
+  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);
+  if (paths.length < 2) {
+    console.error("Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]");
     process.exit(1);
   }
+  const [resultsPath, evalFilePath] = paths;
+  const raw = JSON.parse(await (0, import_promises.readFile)(resultsPath, "utf-8"));
+  const results = Array.isArray(raw) ? raw : [raw];
+  const runtime = await getRuntime(configArg, conditions);
+  try {
+    const mod = await importModule(path.resolve(evalFilePath), import_meta.url);
+    const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
+    if (!evalConfig.scorers || evalConfig.scorers.length === 0) {
+      console.error(`Error: ${evalFilePath} does not export scorers`);
+      process.exit(1);
+    }
+    const rescored = [];
+    for (const resultData of results) {
+      rescored.push(await rescore(resultData, evalConfig.scorers, runtime));
+    }
+    for (const r of rescored) {
+      console.log("\n" + formatTable(r) + "\n");
+    }
+    if (outputPath) {
+      const output = Array.isArray(raw) ? rescored : rescored[0];
+      const outputDir = path.dirname(path.resolve(outputPath));
+      await (0, import_promises.mkdir)(outputDir, { recursive: true });
+      await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
+      console.log(`Rescored results saved to ${outputPath}`);
+    }
+  } finally {
+    await runtime.shutdown().catch(() => {
+    });
+  }
 }
 function collectEvalFiles(p) {
   const resolved = path.resolve(p);
@@ -514,6 +963,35 @@ function formatTable(result) {
   }
   return lines.join("\n");
 }
+function formatMultiRunTable(summary) {
+  const lines = [];
+  const scorerNames = Object.keys(summary.scorers);
+  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), "Scorer".length);
+  const colWidth = 16;
+  lines.push(`Eval: ${summary.workflow} x ${summary.dataset} \u2014 ${summary.runCount} runs`);
+  lines.push(
+    `  ${"Scorer".padEnd(maxNameLen)}  ${"Mean \xB1 Std".padStart(colWidth)}  ${"Min".padStart(8)}  ${"Max".padStart(8)}`
+  );
+  const ruleLen = maxNameLen + 2 + colWidth + 2 + 8 + 2 + 8;
+  lines.push("  " + "\u2500".repeat(ruleLen));
+  for (const name of scorerNames) {
+    const s = summary.scorers[name];
+    const meanStd = `${s.mean.toFixed(3)} \xB1 ${s.std.toFixed(3)}`;
+    lines.push(
+      `  ${name.padEnd(maxNameLen)}  ${meanStd.padStart(colWidth)}  ${s.min.toFixed(3).padStart(8)}  ${s.max.toFixed(3).padStart(8)}`
+    );
+  }
+  if (summary.timing) {
+    lines.push(
+      `  ${"Timing".padEnd(maxNameLen)}  ${((summary.timing.mean / 1e3).toFixed(2) + " \xB1 " + (summary.timing.std / 1e3).toFixed(2) + "s").padStart(colWidth)}`
+    );
+  }
+  const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : "$0.00";
+  const durationStr = (summary.totalDuration / 1e3).toFixed(1);
+  lines.push("");
+  lines.push(`  Total Cost: ${costStr} | Total Duration: ${durationStr}s`);
+  return lines.join("\n");
+}
 async function resolveRuntimeFromConfig(configPath) {
   try {
     const mod = await importModule(configPath, import_meta.url);
@@ -578,10 +1056,11 @@ function parseEvalArgs(args) {
   let outputPath;
   let configArg;
   let conditions = [];
+  let runs = 1;
   const paths = [];
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
-    if (arg === "--output" || arg === "--config" || arg === "--conditions") {
+    if (arg === "--output" || arg === "--config" || arg === "--conditions" || arg === "--runs") {
       if (i + 1 >= args.length) {
         console.error(`Error: ${arg} requires a value`);
         process.exit(1);
@@ -589,6 +1068,7 @@ function parseEvalArgs(args) {
       const value = args[++i];
       if (arg === "--output") outputPath = value;
       else if (arg === "--config") configArg = value;
+      else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
       else
         conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
     } else if (arg.startsWith("--")) {
@@ -600,10 +1080,10 @@ function parseEvalArgs(args) {
       paths.push(arg);
     }
   }
-  return { outputPath, configArg, conditions, paths };
+  return { outputPath, configArg, conditions, runs, paths };
 }
 async function runEvalCommand(args) {
-  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);
+  const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -632,17 +1112,25 @@ async function runEvalCommand(args) {
         let executeWorkflow;
         if (mod.executeWorkflow) {
           executeWorkflow = async (input, rt) => {
-            const { result: result2, cost: trackedCost } = await runtime.trackCost(async () => {
+            const {
+              result,
+              cost: trackedCost,
+              metadata
+            } = await runtime.trackExecution(async () => {
               return mod.executeWorkflow(input, rt);
             });
-            return { output: result2.output, cost: result2.cost ?? trackedCost };
+            return {
+              output: result.output,
+              cost: result.cost ?? trackedCost,
+              metadata: result.metadata ?? metadata
+            };
           };
         } else if (runtime.getWorkflow(evalConfig.workflow)) {
           executeWorkflow = async (input) => {
-            const { result: result2, cost } = await runtime.trackCost(async () => {
+            const { result, cost, metadata } = await runtime.trackExecution(async () => {
               return runtime.execute(evalConfig.workflow, input);
             });
-            return { output: result2, cost };
+            return { output: result, cost, metadata };
           };
         } else {
           console.warn(
@@ -650,9 +1138,25 @@ async function runEvalCommand(args) {
           );
           executeWorkflow = async (input) => ({ output: input });
         }
-        const result = await runEval(evalConfig, executeWorkflow, runtime);
-        results.push(result);
-        console.log("\n" + formatTable(result) + "\n");
+        if (runs > 1) {
+          const { randomUUID: randomUUID4 } = await import("crypto");
+          const runGroupId = randomUUID4();
+          const runResults = [];
+          for (let r = 0; r < runs; r++) {
+            console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
+            const result = await runEval(evalConfig, executeWorkflow, runtime);
+            result.metadata.runGroupId = runGroupId;
+            result.metadata.runIndex = r;
+            runResults.push(result);
+            results.push(result);
+          }
+          const summary = aggregateRuns(runResults);
+          console.log("\n" + formatMultiRunTable(summary) + "\n");
+        } else {
+          const result = await runEval(evalConfig, executeWorkflow, runtime);
+          results.push(result);
+          console.log("\n" + formatTable(result) + "\n");
+        }
       } catch (err) {
         console.error(
           `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`