@axlsdk/eval 0.13.6 → 0.13.8

package/dist/chunk-CMWRFU3P.js

@@ -0,0 +1,653 @@
+// src/bootstrap.ts
+function pairedBootstrapCI(differences, options) {
+  const n = differences.length;
+  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };
+  const nResamples = options?.nResamples ?? 1e3;
+  const alpha = options?.alpha ?? 0.05;
+  const mean = differences.reduce((a, b) => a + b, 0) / n;
+  if (n === 1)
+    return {
+      lower: differences[0],
+      upper: differences[0],
+      mean,
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  if (nResamples <= 0)
+    return {
+      lower: round(mean),
+      upper: round(mean),
+      mean: round(mean),
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();
+  const resampleMeans = new Array(nResamples);
+  for (let r = 0; r < nResamples; r++) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      sum += differences[Math.floor(rng() * n)];
+    }
+    resampleMeans[r] = sum / n;
+  }
+  resampleMeans.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * nResamples);
+  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;
+  let belowZero = 0;
+  for (let i = 0; i < nResamples; i++) {
+    if (resampleMeans[i] < 0) belowZero++;
+    else break;
+  }
+  let aboveZero = 0;
+  for (let i = nResamples - 1; i >= 0; i--) {
+    if (resampleMeans[i] > 0) aboveZero++;
+    else break;
+  }
+  return {
+    lower: round(resampleMeans[Math.max(0, lowerIdx)]),
+    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),
+    mean: round(mean),
+    pRegression: round(belowZero / nResamples),
+    pImprovement: round(aboveZero / nResamples)
+  };
+}
+function round(n) {
+  return Math.round(n * 1e4) / 1e4;
+}
+function xorshift32(seed) {
+  let state = seed | 0 || 1;
+  return () => {
+    state ^= state << 13;
+    state ^= state >> 17;
+    state ^= state << 5;
+    return (state >>> 0) / 4294967296;
+  };
+}
+
+// src/compare.ts
+function round2(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+var DEFAULT_LLM_THRESHOLD = 0.05;
+var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
+var LEGACY_THRESHOLD = 0.1;
+function resolveThreshold(scorerName, options, metadata) {
+  if (typeof options?.thresholds === "number") return options.thresholds;
+  if (options?.thresholds && typeof options.thresholds === "object") {
+    const perScorer = options.thresholds[scorerName];
+    if (perScorer != null) return perScorer;
+  }
+  const scorerTypes = metadata.scorerTypes;
+  if (scorerTypes && scorerName in scorerTypes) {
+    return scorerTypes[scorerName] === "llm" ? DEFAULT_LLM_THRESHOLD : DEFAULT_DETERMINISTIC_THRESHOLD;
+  }
+  return LEGACY_THRESHOLD;
+}
+function evalCompare(baseline, candidate, options) {
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  if (baselineRuns.length === 0 || candidateRuns.length === 0) {
+    throw new Error("Cannot compare empty eval result arrays");
+  }
+  const baselineRef = baselineRuns[0];
+  const candidateRef = candidateRuns[0];
+  if (baselineRef.dataset !== candidateRef.dataset) {
+    throw new Error(
+      `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
+    );
+  }
+  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();
+  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
+    throw new Error(`Cannot compare evals with different scorers`);
+  }
+  const pairedDiffs = {};
+  for (const name of baselineScorerNames) {
+    pairedDiffs[name] = [];
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    const minLength = Math.min(bRun.items.length, cRun.items.length);
+    for (let i = 0; i < minLength; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      if (bItem.error || cItem.error) continue;
+      for (const name of baselineScorerNames) {
+        const bScore = bItem.scores[name];
+        const cScore = cItem.scores[name];
+        if (bScore != null && cScore != null) {
+          pairedDiffs[name].push(cScore - bScore);
+        }
+      }
+    }
+  }
+  const baselineMeans = {};
+  const candidateMeans = {};
+  for (const name of baselineScorerNames) {
+    baselineMeans[name] = baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / baselineRuns.length;
+    candidateMeans[name] = candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / candidateRuns.length;
+  }
+  const scorers = {};
+  for (const name of baselineScorerNames) {
+    const bMeanRaw = baselineMeans[name];
+    const cMeanRaw = candidateMeans[name];
+    const deltaRaw = cMeanRaw - bMeanRaw;
+    const bMean = round2(bMeanRaw);
+    const cMean = round2(cMeanRaw);
+    const delta = round2(deltaRaw);
+    const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
+    const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    const entry = {
+      baselineMean: bMean,
+      candidateMean: cMean,
+      delta,
+      deltaPercent
+    };
+    const diffs = pairedDiffs[name];
+    if (diffs.length >= 2) {
+      const ci = pairedBootstrapCI(diffs);
+      entry.ci = { lower: ci.lower, upper: ci.upper };
+      entry.pRegression = ci.pRegression;
+      entry.pImprovement = ci.pImprovement;
+      entry.n = diffs.length;
+      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
+      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
+    }
+    scorers[name] = entry;
+  }
+  const regressions = [];
+  const improvements = [];
+  const itemCount = Math.min(
+    ...baselineRuns.map((r) => r.items.length),
+    ...candidateRuns.map((r) => r.items.length)
+  );
+  for (let i = 0; i < itemCount; i++) {
+    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);
+    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);
+    if (hasBaselineError || hasCandidateError) continue;
+    for (const name of baselineScorerNames) {
+      const bScores = baselineRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      const cScores = candidateRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      if (bScores.length === 0 || cScores.length === 0) continue;
+      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
+      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
+      const delta = round2(cAvg - bAvg);
+      const threshold = resolveThreshold(name, options, baselineRef.metadata);
+      if (delta < -threshold)
+        regressions.push({
+          itemIndex: i,
+          input: baselineRef.items[i]?.input,
+          scorer: name,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
+          delta
+        });
+      else if (delta > threshold)
+        improvements.push({
+          itemIndex: i,
+          input: baselineRef.items[i]?.input,
+          scorer: name,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
+          delta
+        });
+    }
+  }
+  let timing;
+  const baselineTimings = baselineRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  const candidateTimings = candidateRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (baselineTimings.length > 0 && candidateTimings.length > 0) {
+    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
+    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
+    const delta = round2(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round2((cMean - bMean) / bMean * 100) : 0;
+    timing = { baselineMean: round2(bMean), candidateMean: round2(cMean), delta, deltaPercent };
+  }
+  let cost;
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
+  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
+    const deltaRaw = candidateAvgCost - baselineAvgCost;
+    const delta = round2(deltaRaw);
+    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : 0;
+    cost = {
+      baselineTotal: round2(baselineAvgCost),
+      candidateTotal: round2(candidateAvgCost),
+      delta,
+      deltaPercent
+    };
+  }
+  const parts = [];
+  for (const name of baselineScorerNames) {
+    const s = scorers[name];
+    if (Math.abs(s.delta) > 1e-3) {
+      const direction = s.delta > 0 ? "improves" : "regresses";
+      const sign = s.delta > 0 ? "+" : "";
+      let sigLabel = "";
+      if (s.significant === true) sigLabel = " (significant)";
+      else if (s.significant === false) sigLabel = " (not significant)";
+      parts.push(
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`
+      );
+    }
+  }
+  if (timing && Math.abs(timing.deltaPercent) > 1) {
+    const dir = timing.delta > 0 ? "slower" : "faster";
+    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
+  }
+  if (cost && Math.abs(cost.deltaPercent) > 1) {
+    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
+    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
+  }
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  return {
+    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },
+    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },
+    scorers,
+    timing,
+    cost,
+    regressions,
+    improvements,
+    summary: summaryStr
+  };
+}
+
+// src/scorer.ts
+function normalizeScorerResult(result) {
+  return typeof result === "number" ? { score: result } : result;
+}
+function scorer(config) {
+  return {
+    name: config.name,
+    description: config.description,
+    isLlm: false,
+    score: config.score
+  };
+}
+
+// src/utils.ts
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round3(mean), min: round3(min), max: round3(max), p50: round3(p50), p95: round3(p95) };
+}
+function round3(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+
+// src/runner.ts
+import { randomUUID } from "crypto";
+function parseCost(cost) {
+  const match = cost.match(/^\$?([\d.]+)$/);
+  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
+  return parseFloat(match[1]);
+}
+async function runEval(config, executeWorkflow, runtime) {
+  const startTime = Date.now();
+  const id = randomUUID();
+  const items = await config.dataset.getItems();
+  const concurrency = config.concurrency ?? 5;
+  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    }
+  };
+  const evalItems = new Array(items.length);
+  let totalCost = 0;
+  let budgetExceeded = false;
+  async function processItem(item, itemIndex) {
+    if (budgetExceeded) {
+      evalItems[itemIndex] = {
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Budget exceeded",
+        scores: {}
+      };
+      return;
+    }
+    const evalItem = {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      scores: {}
+    };
+    const itemStart = Date.now();
+    try {
+      const result = await executeWorkflow(item.input, runtime);
+      evalItem.duration = Date.now() - itemStart;
+      evalItem.output = result.output;
+      evalItem.cost = result.cost;
+      if (result.metadata) evalItem.metadata = result.metadata;
+      if (result.cost != null) {
+        totalCost += result.cost;
+      }
+    } catch (err) {
+      evalItem.duration = Date.now() - itemStart;
+      evalItem.error = err instanceof Error ? err.message : String(err);
+      evalItems[itemIndex] = evalItem;
+      return;
+    }
+    if (budgetLimit != null && totalCost > budgetLimit) {
+      budgetExceeded = true;
+    }
+    evalItem.scoreDetails = {};
+    let itemScorerCost = 0;
+    for (const scorer2 of config.scorers) {
+      const scorerStart = Date.now();
+      try {
+        const raw = await scorer2.score(
+          evalItem.output,
+          item.input,
+          item.annotations,
+          scorerContext
+        );
+        const scorerResult = normalizeScorerResult(raw);
+        if (scorerResult.cost != null) {
+          itemScorerCost += scorerResult.cost;
+          totalCost += scorerResult.cost;
+        }
+        const scorerDuration = Date.now() - scorerStart;
+        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
+          evalItem.scorerErrors.push(
+            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
+          );
+          evalItem.scores[scorer2.name] = null;
+          evalItem.scoreDetails[scorer2.name] = {
+            score: null,
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        } else {
+          evalItem.scores[scorer2.name] = round3(scorerResult.score);
+          evalItem.scoreDetails[scorer2.name] = {
+            score: round3(scorerResult.score),
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        }
+      } catch (err) {
+        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
+        if (errCost != null) {
+          itemScorerCost += errCost;
+          totalCost += errCost;
+        }
+        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
+        evalItem.scorerErrors.push(
+          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        evalItem.scores[scorer2.name] = null;
+        evalItem.scoreDetails[scorer2.name] = {
+          score: null,
+          duration: Date.now() - scorerStart,
+          cost: errCost
+        };
+      }
+    }
+    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    evalItems[itemIndex] = evalItem;
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      await processItem(items[currentIndex], currentIndex);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
+  await Promise.all(workers);
+  const failures = evalItems.filter((i) => i.error).length;
+  const scorerNames = config.scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const scorerTypes = {};
+  for (const s of config.scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  const totalModelCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.modelCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [m, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
+      }
+    } else {
+      const itemModels = item.metadata?.models;
+      if (Array.isArray(itemModels)) {
+        for (const m of itemModels) {
+          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const modelsMeta = {};
+  if (totalModelCalls.size > 0) {
+    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
+    modelsMeta.models = sorted.map(([m]) => m);
+    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  }
+  return {
+    id,
+    workflow: config.workflow,
+    dataset: config.dataset.name,
+    metadata: {
+      ...config.metadata,
+      scorerTypes,
+      ...modelsMeta
+    },
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: evalItems,
+    summary: { count: items.length, failures, scorers: scorerStats, timing }
+  };
+}
+
+// src/rescore.ts
+import { randomUUID as randomUUID2 } from "crypto";
+async function rescore(result, scorers, runtime, options) {
+  const startTime = Date.now();
+  const concurrency = options?.concurrency ?? 5;
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    }
+  };
+  const rescored = new Array(result.items.length);
+  let totalCost = 0;
+  async function rescoreItem(original, itemIndex) {
+    if (original.error) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: original.error,
+        scores: {}
+      };
+      return;
+    }
+    const item = {
+      input: original.input,
+      annotations: original.annotations,
+      output: original.output,
+      metadata: original.metadata,
+      scores: {},
+      scoreDetails: {}
+    };
+    let itemScorerCost = 0;
+    for (const scorer2 of scorers) {
+      const scorerStart = Date.now();
+      try {
+        const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
+        const scorerResult = normalizeScorerResult(raw);
+        if (scorerResult.cost != null) {
+          itemScorerCost += scorerResult.cost;
+          totalCost += scorerResult.cost;
+        }
+        const scorerDuration = Date.now() - scorerStart;
+        if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+          if (!item.scorerErrors) item.scorerErrors = [];
+          item.scorerErrors.push(
+            `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`
+          );
+          item.scores[scorer2.name] = null;
+          item.scoreDetails[scorer2.name] = {
+            score: null,
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        } else {
+          item.scores[scorer2.name] = round3(scorerResult.score);
+          item.scoreDetails[scorer2.name] = {
+            score: round3(scorerResult.score),
+            metadata: scorerResult.metadata,
+            duration: scorerDuration,
+            cost: scorerResult.cost
+          };
+        }
+      } catch (err) {
+        const errCost = typeof err?.cost === "number" ? err.cost : void 0;
+        if (errCost != null) {
+          itemScorerCost += errCost;
+          totalCost += errCost;
+        }
+        if (!item.scorerErrors) item.scorerErrors = [];
+        item.scorerErrors.push(
+          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        item.scores[scorer2.name] = null;
+        item.scoreDetails[scorer2.name] = {
+          score: null,
+          duration: Date.now() - scorerStart,
+          cost: errCost
+        };
+      }
+    }
+    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
+    rescored[itemIndex] = item;
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < result.items.length) {
+      const currentIndex = index++;
+      await rescoreItem(result.items[currentIndex], currentIndex);
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, result.items.length) },
+    () => runNext()
+  );
+  await Promise.all(workers);
+  const failures = rescored.filter((i) => i.error).length;
+  const scorerNames = scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  const scorerTypes = {};
+  for (const s of scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  return {
+    id: randomUUID2(),
+    workflow: result.workflow,
+    dataset: result.dataset,
+    metadata: (() => {
+      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
+      return { ...rest, rescored: true, originalId: result.id, scorerTypes };
+    })(),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: rescored,
+    summary: { count: result.items.length, failures, scorers: scorerStats }
+  };
+}
+
+// src/multi-run.ts
+import { randomUUID as randomUUID3 } from "crypto";
+function std(values, mean) {
+  if (values.length <= 1) return 0;
+  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);
+  return Math.sqrt(sumSqDiffs / (values.length - 1));
+}
+function aggregateRuns(runs) {
+  if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
+  const runGroupId = runs[0].metadata.runGroupId ?? randomUUID3();
+  const { workflow, dataset } = runs[0];
+  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
+  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+  const scorerNames = Object.keys(runs[0].summary.scorers);
+  const scorers = {};
+  for (const name of scorerNames) {
+    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
+    const minMean = Math.min(...means);
+    const maxMean = Math.max(...means);
+    scorers[name] = {
+      mean: round3(meanOfMeans),
+      std: round3(std(means, meanOfMeans)),
+      min: round3(minMean),
+      max: round3(maxMean)
+    };
+  }
+  let timing;
+  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (timingMeans.length === runs.length && timingMeans.length > 0) {
+    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
+    timing = {
+      mean: round3(meanTiming),
+      std: round3(std(timingMeans, meanTiming))
+    };
+  }
+  return {
+    runGroupId,
+    runCount: runs.length,
+    workflow,
+    dataset,
+    totalCost: round3(totalCost),
+    totalDuration: round3(totalDuration),
+    scorers,
+    timing
+  };
+}
+
+export {
+  pairedBootstrapCI,
+  evalCompare,
+  normalizeScorerResult,
+  scorer,
+  runEval,
+  rescore,
+  aggregateRuns
+};
+//# sourceMappingURL=chunk-CMWRFU3P.js.map