@axlsdk/eval 0.13.6 → 0.13.8

package/dist/index.cjs

@@ -30,11 +30,14 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  aggregateRuns: () => aggregateRuns,
   dataset: () => dataset,
   defineEval: () => defineEval,
   evalCompare: () => evalCompare,
   llmScorer: () => llmScorer,
   normalizeScorerResult: () => normalizeScorerResult,
+  pairedBootstrapCI: () => pairedBootstrapCI,
+  rescore: () => rescore,
   runEval: () => runEval,
   scorer: () => scorer
 });
@@ -159,8 +162,7 @@ function defineEval(config) {
   return config;
 }
 
-// src/runner.ts
-var import_node_crypto = require("crypto");
+// src/utils.ts
 function computeStats(scores) {
   if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
   const sorted = [...scores].sort((a, b) => a - b);
@@ -174,6 +176,9 @@ function computeStats(scores) {
 function round(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+
+// src/runner.ts
+var import_node_crypto = require("crypto");
 function parseCost(cost) {
   const match = cost.match(/^\$?([\d.]+)$/);
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
@@ -221,6 +226,7 @@ async function runEval(config, executeWorkflow, runtime) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.output = result.output;
       evalItem.cost = result.cost;
+      if (result.metadata) evalItem.metadata = result.metadata;
       if (result.cost != null) {
         totalCost += result.cost;
       }
@@ -310,11 +316,42 @@ async function runEval(config, executeWorkflow, runtime) {
   }
   const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
   const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const scorerTypes = {};
+  for (const s of config.scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  const totalModelCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.modelCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [m, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
+      }
+    } else {
+      const itemModels = item.metadata?.models;
+      if (Array.isArray(itemModels)) {
+        for (const m of itemModels) {
+          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const modelsMeta = {};
+  if (totalModelCalls.size > 0) {
+    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
+    modelsMeta.models = sorted.map(([m]) => m);
+    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  }
   return {
     id,
     workflow: config.workflow,
     dataset: config.dataset.name,
-    metadata: config.metadata ?? {},
+    metadata: {
+      ...config.metadata,
+      scorerTypes,
+      ...modelsMeta
+    },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
     duration: Date.now() - startTime,
@@ -323,76 +360,223 @@ async function runEval(config, executeWorkflow, runtime) {
   };
 }
 
-// src/compare.ts
+// src/bootstrap.ts
+function pairedBootstrapCI(differences, options) {
+  const n = differences.length;
+  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };
+  const nResamples = options?.nResamples ?? 1e3;
+  const alpha = options?.alpha ?? 0.05;
+  const mean = differences.reduce((a, b) => a + b, 0) / n;
+  if (n === 1)
+    return {
+      lower: differences[0],
+      upper: differences[0],
+      mean,
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  if (nResamples <= 0)
+    return {
+      lower: round2(mean),
+      upper: round2(mean),
+      mean: round2(mean),
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();
+  const resampleMeans = new Array(nResamples);
+  for (let r = 0; r < nResamples; r++) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      sum += differences[Math.floor(rng() * n)];
+    }
+    resampleMeans[r] = sum / n;
+  }
+  resampleMeans.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * nResamples);
+  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;
+  let belowZero = 0;
+  for (let i = 0; i < nResamples; i++) {
+    if (resampleMeans[i] < 0) belowZero++;
+    else break;
+  }
+  let aboveZero = 0;
+  for (let i = nResamples - 1; i >= 0; i--) {
+    if (resampleMeans[i] > 0) aboveZero++;
+    else break;
+  }
+  return {
+    lower: round2(resampleMeans[Math.max(0, lowerIdx)]),
+    upper: round2(resampleMeans[Math.min(nResamples - 1, upperIdx)]),
+    mean: round2(mean),
+    pRegression: round2(belowZero / nResamples),
+    pImprovement: round2(aboveZero / nResamples)
+  };
+}
 function round2(n) {
-  return Math.round(n * 100) / 100;
+  return Math.round(n * 1e4) / 1e4;
+}
+function xorshift32(seed) {
+  let state = seed | 0 || 1;
+  return () => {
+    state ^= state << 13;
+    state ^= state >> 17;
+    state ^= state << 5;
+    return (state >>> 0) / 4294967296;
+  };
+}
+
+// src/compare.ts
+function round3(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+var DEFAULT_LLM_THRESHOLD = 0.05;
+var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
+var LEGACY_THRESHOLD = 0.1;
+function resolveThreshold(scorerName, options, metadata) {
+  if (typeof options?.thresholds === "number") return options.thresholds;
+  if (options?.thresholds && typeof options.thresholds === "object") {
+    const perScorer = options.thresholds[scorerName];
+    if (perScorer != null) return perScorer;
+  }
+  const scorerTypes = metadata.scorerTypes;
+  if (scorerTypes && scorerName in scorerTypes) {
+    return scorerTypes[scorerName] === "llm" ? DEFAULT_LLM_THRESHOLD : DEFAULT_DETERMINISTIC_THRESHOLD;
+  }
+  return LEGACY_THRESHOLD;
 }
-function evalCompare(baseline, candidate) {
-  if (baseline.dataset !== candidate.dataset) {
+function evalCompare(baseline, candidate, options) {
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  if (baselineRuns.length === 0 || candidateRuns.length === 0) {
+    throw new Error("Cannot compare empty eval result arrays");
+  }
+  const baselineRef = baselineRuns[0];
+  const candidateRef = candidateRuns[0];
+  if (baselineRef.dataset !== candidateRef.dataset) {
     throw new Error(
-      `Cannot compare evals from different datasets: "${baseline.dataset}" vs "${candidate.dataset}"`
+      `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
     );
   }
-  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();
-  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();
+  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();
   if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
     throw new Error(`Cannot compare evals with different scorers`);
   }
+  const pairedDiffs = {};
+  for (const name of baselineScorerNames) {
+    pairedDiffs[name] = [];
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    const minLength = Math.min(bRun.items.length, cRun.items.length);
+    for (let i = 0; i < minLength; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      if (bItem.error || cItem.error) continue;
+      for (const name of baselineScorerNames) {
+        const bScore = bItem.scores[name];
+        const cScore = cItem.scores[name];
+        if (bScore != null && cScore != null) {
+          pairedDiffs[name].push(cScore - bScore);
+        }
+      }
+    }
+  }
+  const baselineMeans = {};
+  const candidateMeans = {};
+  for (const name of baselineScorerNames) {
+    baselineMeans[name] = baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / baselineRuns.length;
+    candidateMeans[name] = candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / candidateRuns.length;
+  }
   const scorers = {};
   for (const name of baselineScorerNames) {
-    const bMean = baseline.summary.scorers[name].mean;
-    const cMean = candidate.summary.scorers[name].mean;
-    const delta = round2(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round2(delta / bMean * 100) : 0;
-    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+    const bMeanRaw = baselineMeans[name];
+    const cMeanRaw = candidateMeans[name];
+    const deltaRaw = cMeanRaw - bMeanRaw;
+    const bMean = round3(bMeanRaw);
+    const cMean = round3(cMeanRaw);
+    const delta = round3(deltaRaw);
+    const deltaPercent = bMeanRaw > 0 ? round3(deltaRaw / bMeanRaw * 100) : 0;
+    const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    const entry = {
+      baselineMean: bMean,
+      candidateMean: cMean,
+      delta,
+      deltaPercent
+    };
+    const diffs = pairedDiffs[name];
+    if (diffs.length >= 2) {
+      const ci = pairedBootstrapCI(diffs);
+      entry.ci = { lower: ci.lower, upper: ci.upper };
+      entry.pRegression = ci.pRegression;
+      entry.pImprovement = ci.pImprovement;
+      entry.n = diffs.length;
+      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
+      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
+    }
+    scorers[name] = entry;
   }
   const regressions = [];
   const improvements = [];
-  const minLength = Math.min(baseline.items.length, candidate.items.length);
-  for (let i = 0; i < minLength; i++) {
-    const bItem = baseline.items[i];
-    const cItem = candidate.items[i];
-    if (bItem.error || cItem.error) continue;
+  const itemCount = Math.min(
+    ...baselineRuns.map((r) => r.items.length),
+    ...candidateRuns.map((r) => r.items.length)
+  );
+  for (let i = 0; i < itemCount; i++) {
+    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);
+    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);
+    if (hasBaselineError || hasCandidateError) continue;
     for (const name of baselineScorerNames) {
-      const bScore = bItem.scores[name];
-      const cScore = cItem.scores[name];
-      if (bScore == null || cScore == null) continue;
-      const delta = round2(cScore - bScore);
-      if (delta < -0.1)
+      const bScores = baselineRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      const cScores = candidateRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      if (bScores.length === 0 || cScores.length === 0) continue;
+      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
+      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
+      const delta = round3(cAvg - bAvg);
+      const threshold = resolveThreshold(name, options, baselineRef.metadata);
+      if (delta < -threshold)
         regressions.push({
           itemIndex: i,
-          input: bItem.input,
+          input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
+          baselineScore: round3(bAvg),
+          candidateScore: round3(cAvg),
           delta
         });
-      else if (delta > 0.1)
+      else if (delta > threshold)
         improvements.push({
           itemIndex: i,
-          input: bItem.input,
+          input: baselineRef.items[i]?.input,
           scorer: name,
-          baselineScore: bScore,
-          candidateScore: cScore,
+          baselineScore: round3(bAvg),
+          candidateScore: round3(cAvg),
           delta
         });
     }
   }
   let timing;
-  if (baseline.summary.timing && candidate.summary.timing) {
-    const bMean = baseline.summary.timing.mean;
-    const cMean = candidate.summary.timing.mean;
-    const delta = round2(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round2(delta / bMean * 100) : 0;
-    timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+  const baselineTimings = baselineRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  const candidateTimings = candidateRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (baselineTimings.length > 0 && candidateTimings.length > 0) {
+    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
+    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
+    const delta = round3(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round3((cMean - bMean) / bMean * 100) : 0;
+    timing = { baselineMean: round3(bMean), candidateMean: round3(cMean), delta, deltaPercent };
   }
   let cost;
-  if (baseline.totalCost > 0 || candidate.totalCost > 0) {
-    const delta = round2(candidate.totalCost - baseline.totalCost);
-    const deltaPercent = baseline.totalCost > 0 ? round2(delta / baseline.totalCost * 100) : 0;
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
+  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
+    const deltaRaw = candidateAvgCost - baselineAvgCost;
+    const delta = round3(deltaRaw);
+    const deltaPercent = baselineAvgCost > 0 ? round3(deltaRaw / baselineAvgCost * 100) : 0;
     cost = {
-      baselineTotal: baseline.totalCost,
-      candidateTotal: candidate.totalCost,
+      baselineTotal: round3(baselineAvgCost),
+      candidateTotal: round3(candidateAvgCost),
       delta,
       deltaPercent
     };
@@ -403,8 +587,11 @@ function evalCompare(baseline, candidate) {
     if (Math.abs(s.delta) > 1e-3) {
       const direction = s.delta > 0 ? "improves" : "regresses";
       const sign = s.delta > 0 ? "+" : "";
+      let sigLabel = "";
+      if (s.significant === true) sigLabel = " (significant)";
+      else if (s.significant === false) sigLabel = " (not significant)";
       parts.push(
-        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`
       );
     }
   }
@@ -416,10 +603,10 @@ function evalCompare(baseline, candidate) {
     const dir = cost.delta > 0 ? "more expensive" : "cheaper";
     parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
   }
-  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
   return {
-    baseline: { id: baseline.id, metadata: baseline.metadata },
-    candidate: { id: candidate.id, metadata: candidate.metadata },
+    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },
+    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },
     scorers,
     timing,
     cost,
@@ -428,13 +615,192 @@ function evalCompare(baseline, candidate) {
     summary: summaryStr
   };
 }
+
+// src/rescore.ts
+var import_node_crypto2 = require("crypto");
+async function rescore(result, scorers, runtime, options) {
+  const startTime = Date.now();
+  const concurrency = options?.concurrency ?? 5;
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    }
+  };
+  const rescored = new Array(result.items.length);
+  let totalCost = 0;
+  async function rescoreItem(original, itemIndex) {
+    if (original.error) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: original.error,
+        scores: {}
+      };
+      return;
+    }
+    const item = {
+      input: original.input,
+      annotations: original.annotations,
+      output: original.output,
+      metadata: original.metadata,
+      scores: {},
+      scoreDetails: {}
+    };
+    let itemScorerCost = 0;
+    for (const scorer2 of scorers) {
+      const scorerStart = Date.now();
+      try {
+        const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
+        const scorerResult = normalizeScorerResult(raw);
+        if (scorerResult.cost != null) {
+          itemScorerCost += scorerResult.cost;
+          totalCost += scorerResult.cost;
+        }
+        const scorerDuration = Date.now() - scorerStart;
+        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+          if (!item.scorerErrors) item.scorerErrors = [];
+          item.scorerErrors.push(
+            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
+          );
+          item.scores[scorer2.name] = null;
+          item.scoreDetails[scorer2.name] = {
+            score: null,
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        } else {
+          item.scores[scorer2.name] = round(scorerResult.score);
+          item.scoreDetails[scorer2.name] = {
+            score: round(scorerResult.score),
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        }
+      } catch (err) {
+        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
+        if (errCost != null) {
+          itemScorerCost += errCost;
+          totalCost += errCost;
+        }
+        if (!item.scorerErrors) item.scorerErrors = [];
+        item.scorerErrors.push(
+          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        item.scores[scorer2.name] = null;
+        item.scoreDetails[scorer2.name] = {
+          score: null,
+          duration: Date.now() - scorerStart,
+          cost: errCost
+        };
+      }
+    }
+    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    rescored[itemIndex] = item;
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < result.items.length) {
+      const currentIndex = index++;
+      await rescoreItem(result.items[currentIndex], currentIndex);
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, result.items.length) },
+    () => runNext()
+  );
+  await Promise.all(workers);
+  const failures = rescored.filter((i) => i.error).length;
+  const scorerNames = scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  const scorerTypes = {};
+  for (const s of scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  return {
+    id: (0, import_node_crypto2.randomUUID)(),
+    workflow: result.workflow,
+    dataset: result.dataset,
+    metadata: (() => {
+      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      return { ...rest, rescored: true, originalId: result.id, scorerTypes };
+    })(),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: rescored,
+    summary: { count: result.items.length, failures, scorers: scorerStats }
+  };
+}
+
+// src/multi-run.ts
+var import_node_crypto3 = require("crypto");
+function std(values, mean) {
+  if (values.length <= 1) return 0;
+  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);
+  return Math.sqrt(sumSqDiffs / (values.length - 1));
+}
+function aggregateRuns(runs) {
+  if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
+  const runGroupId = runs[0].metadata.runGroupId ?? (0, import_node_crypto3.randomUUID)();
+  const { workflow, dataset: dataset2 } = runs[0];
+  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
+  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+  const scorerNames = Object.keys(runs[0].summary.scorers);
+  const scorers = {};
+  for (const name of scorerNames) {
+    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
+    const minMean = Math.min(...means);
+    const maxMean = Math.max(...means);
+    scorers[name] = {
+      mean: round(meanOfMeans),
+      std: round(std(means, meanOfMeans)),
+      min: round(minMean),
+      max: round(maxMean)
+    };
+  }
+  let timing;
+  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (timingMeans.length === runs.length && timingMeans.length > 0) {
+    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
+    timing = {
+      mean: round(meanTiming),
+      std: round(std(timingMeans, meanTiming))
+    };
+  }
+  return {
+    runGroupId,
+    runCount: runs.length,
+    workflow,
+    dataset: dataset2,
+    totalCost: round(totalCost),
+    totalDuration: round(totalDuration),
+    scorers,
+    timing
+  };
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  aggregateRuns,
   dataset,
   defineEval,
   evalCompare,
   llmScorer,
   normalizeScorerResult,
+  pairedBootstrapCI,
+  rescore,
   runEval,
   scorer
 });