@axlsdk/eval 0.23.3 → 0.24.1

package/dist/chunk-AFUNZE7W.js

@@ -1,966 +0,0 @@
-// src/bootstrap.ts
-function pairedBootstrapCI(differences, options) {
-  const n = differences.length;
-  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };
-  const nResamples = options?.nResamples ?? 1e3;
-  const alpha = options?.alpha ?? 0.05;
-  const mean = differences.reduce((a, b) => a + b, 0) / n;
-  if (n === 1)
-    return {
-      lower: differences[0],
-      upper: differences[0],
-      mean,
-      pRegression: mean < 0 ? 1 : 0,
-      pImprovement: mean > 0 ? 1 : 0
-    };
-  if (nResamples <= 0)
-    return {
-      lower: round(mean),
-      upper: round(mean),
-      mean: round(mean),
-      pRegression: mean < 0 ? 1 : 0,
-      pImprovement: mean > 0 ? 1 : 0
-    };
-  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();
-  const resampleMeans = new Array(nResamples);
-  for (let r = 0; r < nResamples; r++) {
-    let sum = 0;
-    for (let i = 0; i < n; i++) {
-      sum += differences[Math.floor(rng() * n)];
-    }
-    resampleMeans[r] = sum / n;
-  }
-  resampleMeans.sort((a, b) => a - b);
-  const lowerIdx = Math.floor(alpha / 2 * nResamples);
-  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;
-  let belowZero = 0;
-  for (let i = 0; i < nResamples; i++) {
-    if (resampleMeans[i] < 0) belowZero++;
-    else break;
-  }
-  let aboveZero = 0;
-  for (let i = nResamples - 1; i >= 0; i--) {
-    if (resampleMeans[i] > 0) aboveZero++;
-    else break;
-  }
-  return {
-    lower: round(resampleMeans[Math.max(0, lowerIdx)]),
-    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),
-    mean: round(mean),
-    pRegression: round(belowZero / nResamples),
-    pImprovement: round(aboveZero / nResamples)
-  };
-}
-function round(n) {
-  return Math.round(n * 1e4) / 1e4;
-}
-function xorshift32(seed) {
-  let state = seed | 0 || 1;
-  return () => {
-    state ^= state << 13;
-    state ^= state >> 17;
-    state ^= state << 5;
-    return (state >>> 0) / 4294967296;
-  };
-}
-
-// src/utils.ts
-function computeStats(scores) {
-  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
-  const sorted = [...scores].sort((a, b) => a - b);
-  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  const p50 = sorted[Math.floor(sorted.length * 0.5)];
-  const p95 = sorted[Math.floor(sorted.length * 0.95)];
-  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
-}
-function round2(n) {
-  return Math.round(n * 1e3) / 1e3;
-}
-function scorerCounts(items, name) {
-  let scored = 0;
-  let failed = 0;
-  let skipped = 0;
-  for (const i of items) {
-    if (i.error) continue;
-    if (i.scores[name] != null) scored++;
-    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;
-    else if (i.scoreDetails?.[name]?.duration != null) failed++;
-  }
-  return { scored, failed, skipped };
-}
-function evaluateScorerTolerance(scored, failed, type, limit) {
-  const attempted = scored + failed;
-  const rate = attempted === 0 ? 0 : failed / attempted;
-  const exceeds = attempted > 0 && (type === "deterministic" ? failed > 0 : rate > limit);
-  return { attempted, rate, exceeds, zeroSample: attempted === 0 };
-}
-async function mapWithConcurrency(items, concurrency, task) {
-  const results = new Array(items.length);
-  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
-  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);
-  let next = 0;
-  async function runNext() {
-    while (next < items.length) {
-      const i = next++;
-      results[i] = await task(items[i], i);
-    }
-  }
-  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
-  return results;
-}
-
-// src/compare.ts
-var DEFAULT_LLM_THRESHOLD = 0.05;
-var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
-var LEGACY_THRESHOLD = 0.1;
-function resolveThreshold(scorerName, options, metadata) {
-  if (typeof options?.thresholds === "number") return options.thresholds;
-  if (options?.thresholds && typeof options.thresholds === "object") {
-    const perScorer = options.thresholds[scorerName];
-    if (perScorer != null) return perScorer;
-  }
-  const scorerTypes = metadata.scorerTypes;
-  if (scorerTypes && scorerName in scorerTypes) {
-    return scorerTypes[scorerName] === "llm" ? DEFAULT_LLM_THRESHOLD : DEFAULT_DETERMINISTIC_THRESHOLD;
-  }
-  return LEGACY_THRESHOLD;
-}
-function evalCompare(baseline, candidate, options) {
-  const baselineRunsRaw = Array.isArray(baseline) ? baseline : [baseline];
-  const candidateRunsRaw = Array.isArray(candidate) ? candidate : [candidate];
-  if (baselineRunsRaw.length === 0 || candidateRunsRaw.length === 0) {
-    throw new Error("Cannot compare empty eval result arrays");
-  }
-  const runCount = Math.min(baselineRunsRaw.length, candidateRunsRaw.length);
-  const baselineRuns = baselineRunsRaw.slice(0, runCount);
-  const candidateRuns = candidateRunsRaw.slice(0, runCount);
-  const baselineRef = baselineRunsRaw[0];
-  const candidateRef = candidateRunsRaw[0];
-  if (baselineRef.dataset !== candidateRef.dataset) {
-    throw new Error(
-      `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
-    );
-  }
-  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();
-  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();
-  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
-    throw new Error(`Cannot compare evals with different scorers`);
-  }
-  const pairedDiffs = {};
-  for (const name of baselineScorerNames) {
-    pairedDiffs[name] = [];
-  }
-  for (let r = 0; r < runCount; r++) {
-    const bRun = baselineRuns[r];
-    const cRun = candidateRuns[r];
-    const minLength = Math.min(bRun.items.length, cRun.items.length);
-    for (let i = 0; i < minLength; i++) {
-      const bItem = bRun.items[i];
-      const cItem = cRun.items[i];
-      if (bItem.error || cItem.error) continue;
-      for (const name of baselineScorerNames) {
-        const bScore = bItem.scores[name];
-        const cScore = cItem.scores[name];
-        if (bScore != null && cScore != null) {
-          pairedDiffs[name].push(cScore - bScore);
-        }
-      }
-    }
-  }
-  const baselineMeans = {};
-  const candidateMeans = {};
-  for (const name of baselineScorerNames) {
-    baselineMeans[name] = baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / baselineRuns.length;
-    candidateMeans[name] = candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / candidateRuns.length;
-  }
-  const scorers = {};
-  for (const name of baselineScorerNames) {
-    const bMeanRaw = baselineMeans[name];
-    const cMeanRaw = candidateMeans[name];
-    const deltaRaw = cMeanRaw - bMeanRaw;
-    const bMean = round2(bMeanRaw);
-    const cMean = round2(cMeanRaw);
-    const delta = round2(deltaRaw);
-    const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
-    const threshold = resolveThreshold(name, options, baselineRef.metadata);
-    let baselineScored = 0;
-    let baselineFailed = 0;
-    let baselineSkipped = 0;
-    for (const run of baselineRuns) {
-      const c = scorerCounts(run.items, name);
-      baselineScored += c.scored;
-      baselineFailed += c.failed;
-      baselineSkipped += c.skipped;
-    }
-    let candidateScored = 0;
-    let candidateFailed = 0;
-    let candidateSkipped = 0;
-    for (const run of candidateRuns) {
-      const c = scorerCounts(run.items, name);
-      candidateScored += c.scored;
-      candidateFailed += c.failed;
-      candidateSkipped += c.skipped;
-    }
-    const entry = {
-      baselineMean: bMean,
-      candidateMean: cMean,
-      delta,
-      deltaPercent,
-      baselineScored,
-      baselineFailed,
-      candidateScored,
-      candidateFailed,
-      baselineSkipped,
-      candidateSkipped
-    };
-    const diffs = pairedDiffs[name];
-    entry.n = diffs.length;
-    if (diffs.length >= 2) {
-      const ci = pairedBootstrapCI(diffs);
-      entry.ci = { lower: ci.lower, upper: ci.upper };
-      entry.pRegression = ci.pRegression;
-      entry.pImprovement = ci.pImprovement;
-      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
-      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
-    }
-    scorers[name] = entry;
-  }
-  const regressions = [];
-  const improvements = [];
-  const itemCount = Math.min(
-    ...baselineRuns.map((r) => r.items.length),
-    ...candidateRuns.map((r) => r.items.length)
-  );
-  for (let i = 0; i < itemCount; i++) {
-    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);
-    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);
-    if (hasBaselineError || hasCandidateError) continue;
-    for (const name of baselineScorerNames) {
-      const bScores = baselineRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
-      const cScores = candidateRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
-      if (bScores.length === 0 || cScores.length === 0) continue;
-      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
-      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
-      const delta = round2(cAvg - bAvg);
-      const threshold = resolveThreshold(name, options, baselineRef.metadata);
-      if (delta < -threshold)
-        regressions.push({
-          itemIndex: i,
-          input: baselineRef.items[i]?.input,
-          scorer: name,
-          baselineScore: round2(bAvg),
-          candidateScore: round2(cAvg),
-          delta
-        });
-      else if (delta > threshold)
-        improvements.push({
-          itemIndex: i,
-          input: baselineRef.items[i]?.input,
-          scorer: name,
-          baselineScore: round2(bAvg),
-          candidateScore: round2(cAvg),
-          delta
-        });
-    }
-  }
-  let timing;
-  const baselineTimings = baselineRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
-  const candidateTimings = candidateRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
-  if (baselineTimings.length > 0 && candidateTimings.length > 0) {
-    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
-    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
-    const delta = round2(cMean - bMean);
-    const deltaPercent = bMean > 0 ? round2((cMean - bMean) / bMean * 100) : 0;
-    timing = { baselineMean: round2(bMean), candidateMean: round2(cMean), delta, deltaPercent };
-  }
-  let cost;
-  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
-  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
-  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
-    const deltaRaw = candidateAvgCost - baselineAvgCost;
-    const delta = round2(deltaRaw);
-    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : 0;
-    cost = {
-      baselineTotal: round2(baselineAvgCost),
-      candidateTotal: round2(candidateAvgCost),
-      delta,
-      deltaPercent
-    };
-  }
-  const parts = [];
-  for (const name of baselineScorerNames) {
-    const s = scorers[name];
-    if (Math.abs(s.delta) > 1e-3) {
-      const direction = s.delta > 0 ? "improves" : "regresses";
-      const sign = s.delta > 0 ? "+" : "";
-      let sigLabel = "";
-      if (s.significant === true) sigLabel = " (significant)";
-      else if (s.significant === false) sigLabel = " (not significant)";
-      parts.push(
-        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`
-      );
-    }
-  }
-  if (timing && Math.abs(timing.deltaPercent) > 1) {
-    const dir = timing.delta > 0 ? "slower" : "faster";
-    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  if (cost && Math.abs(cost.deltaPercent) > 1) {
-    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
-    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
-  const baselinePartial = detectPartial(baselineRunsRaw);
-  const candidatePartial = detectPartial(candidateRunsRaw);
-  return {
-    baseline: {
-      id: baselineRef.id,
-      metadata: baselineRef.metadata,
-      runCount,
-      ...baselinePartial ? { partial: baselinePartial } : {}
-    },
-    candidate: {
-      id: candidateRef.id,
-      metadata: candidateRef.metadata,
-      runCount,
-      ...candidatePartial ? { partial: candidatePartial } : {}
-    },
-    scorers,
-    timing,
-    cost,
-    regressions,
-    improvements,
-    summary: summaryStr
-  };
-}
-function detectPartial(runs) {
-  if (runs.length === 0) return void 0;
-  let attempted;
-  for (const r of runs) {
-    const v = r.metadata?.batchAttempted;
-    if (typeof v === "number" && Number.isFinite(v)) {
-      attempted = v;
-      break;
-    }
-  }
-  if (attempted === void 0) return void 0;
-  if (runs.length >= attempted) return void 0;
-  return { completed: runs.length, attempted };
-}
-function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
-  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
-  for (const name of Object.keys(comparison.scorers)) {
-    const s = comparison.scorers[name];
-    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
-    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
-    for (const side of ["baseline", "candidate"]) {
-      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
-      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
-      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
-      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
-      if (verdict.zeroSample) {
-        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
-        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
-      }
-      if (verdict.exceeds) {
-        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
-        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
-      }
-    }
-  }
-  return null;
-}
-
-// src/scorer.ts
-function normalizeScorerResult(result) {
-  return typeof result === "number" ? { score: result } : result;
-}
-function attachScorerErrorCost(err, cost) {
-  if (err && typeof err === "object" && typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
-    err.cost = cost;
-  }
-}
-function extractScorerErrorCost(err) {
-  const v = err && typeof err === "object" ? err.cost : void 0;
-  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
-}
-function scorer(config) {
-  return {
-    name: config.name,
-    description: config.description,
-    isLlm: false,
-    score: config.score,
-    applies: config.applies
-  };
-}
-
-// src/score-item.ts
-async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
-  const scoreDetails = item.scoreDetails ??= {};
-  for (const s of scorers) {
-    item.scores[s.name] = null;
-    scoreDetails[s.name] = { score: null };
-  }
-  let itemScorerCost = 0;
-  const scorerErrorsByName = {};
-  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer2) => {
-    if (signal?.aborted) return;
-    const scorerStart = Date.now();
-    try {
-      if (scorer2.applies && !scorer2.applies(item.output, item.input, item.annotations)) {
-        item.scores[scorer2.name] = null;
-        scoreDetails[scorer2.name] = { score: null, skipped: true };
-        return;
-      }
-      const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
-      const scorerResult = normalizeScorerResult(raw);
-      if (scorerResult.cost != null) {
-        itemScorerCost += scorerResult.cost;
-      }
-      const scorerDuration = Date.now() - scorerStart;
-      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
-        item.scores[scorer2.name] = null;
-        scoreDetails[scorer2.name] = {
-          score: null,
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
-      } else {
-        item.scores[scorer2.name] = round2(scorerResult.score);
-        scoreDetails[scorer2.name] = {
-          score: round2(scorerResult.score),
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
-      }
-    } catch (err) {
-      if (err?.name === "AbortError") return;
-      const errCost = extractScorerErrorCost(err);
-      if (errCost != null) itemScorerCost += errCost;
-      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
-      item.scores[scorer2.name] = null;
-      scoreDetails[scorer2.name] = {
-        score: null,
-        duration: Date.now() - scorerStart,
-        cost: errCost
-      };
-    }
-  });
-  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
-  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
-  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
-  return itemScorerCost;
-}
-
-// src/runner.ts
-import { randomUUID } from "crypto";
-function parseCost(cost) {
-  const match = cost.match(/^\$?([\d.]+)$/);
-  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
-  return parseFloat(match[1]);
-}
-function extractUserCost(result, label = "executeWorkflow") {
-  if (result === null || typeof result !== "object") return void 0;
-  const raw = result.cost;
-  if (raw === void 0) return void 0;
-  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
-  console.warn(
-    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
-  );
-  return void 0;
-}
-function extractUserMetadata(result, label = "executeWorkflow") {
-  if (result === null || typeof result !== "object") return void 0;
-  const meta = result.metadata;
-  if (meta === void 0) return void 0;
-  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
-    console.warn(
-      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
-    );
-    return void 0;
-  }
-  const proto = Object.getPrototypeOf(meta);
-  if (proto !== Object.prototype && proto !== null) {
-    console.warn(
-      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
-    );
-    return void 0;
-  }
-  return meta;
-}
-function mergeMetadata(tracked, user) {
-  const merged = { ...tracked, ...user };
-  for (const [list, counts] of [
-    ["models", "modelCallCounts"],
-    ["workflows", "workflowCallCounts"]
-  ]) {
-    if (user && Object.hasOwn(user, list) && !Object.hasOwn(user, counts)) {
-      delete merged[counts];
-    }
-  }
-  return merged;
-}
-async function runEval(config, executeWorkflow, runtime, options) {
-  const startTime = Date.now();
-  const id = randomUUID();
-  const items = await config.dataset.getItems();
-  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
-  const concurrency = config.concurrency ?? 5;
-  const scorerConcurrency = config.scorerConcurrency ?? 5;
-  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
-  const scorerContext = {
-    resolveProvider: (uri) => {
-      if (typeof runtime.resolveProvider !== "function") {
-        throw new Error(
-          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
-        );
-      }
-      return runtime.resolveProvider(uri);
-    },
-    signal: options?.signal
-  };
-  const evalItems = new Array(items.length);
-  let totalCost = 0;
-  let budgetExceeded = false;
-  const runCallSamples = /* @__PURE__ */ new Map();
-  function absorbTiming(rollup, evalItem) {
-    if (!rollup) return;
-    const perItem = {};
-    for (const [model, bucket] of Object.entries(rollup)) {
-      const sums = {
-        calls: bucket.calls,
-        queuedMs: bucket.queuedMs,
-        retryMs: bucket.retryMs,
-        wireMs: bucket.wireMs,
-        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
-      };
-      perItem[model] = sums;
-      const samples = bucket.samples;
-      if (!samples) continue;
-      const pooled = runCallSamples.get(model) ?? [];
-      for (const sample of samples) pooled.push(sample);
-      runCallSamples.set(model, pooled);
-    }
-    evalItem.timing = perItem;
-  }
-  async function processItem(item, itemIndex) {
-    if (options?.signal?.aborted) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Cancelled",
-        scores: {}
-      };
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    if (budgetExceeded) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Budget exceeded",
-        scores: {}
-      };
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    const evalItem = {
-      input: item.input,
-      annotations: item.annotations,
-      output: null,
-      scores: {}
-    };
-    const itemStart = Date.now();
-    try {
-      if (options?.captureTraces) {
-        const tracked = await runtime.trackExecution(
-          async () => executeWorkflow(item.input, runtime),
-          { captureTraces: true, captureTimingSamples: true }
-        );
-        evalItem.duration = Date.now() - itemStart;
-        evalItem.output = tracked.result.output;
-        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
-        if (tracked.unpriced) evalItem.unpriced = true;
-        evalItem.metadata = mergeMetadata(tracked.metadata, extractUserMetadata(tracked.result));
-        absorbTiming(tracked.modelTiming, evalItem);
-        if (tracked.traces && tracked.traces.length > 0) {
-          evalItem.traces = tracked.traces;
-        }
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
-        }
-      } else {
-        let trackedTiming;
-        let trackedMetadata;
-        let result;
-        if (typeof runtime.trackExecution === "function") {
-          const tracked = await runtime.trackExecution(
-            async () => executeWorkflow(item.input, runtime),
-            { captureTraces: false, captureTimingSamples: true }
-          );
-          result = tracked.result;
-          trackedTiming = tracked.modelTiming;
-          trackedMetadata = tracked.metadata;
-        } else {
-          result = await executeWorkflow(item.input, runtime);
-        }
-        evalItem.duration = Date.now() - itemStart;
-        evalItem.output = result.output;
-        evalItem.cost = extractUserCost(result);
-        const meta = extractUserMetadata(result);
-        if (trackedMetadata || meta) evalItem.metadata = mergeMetadata(trackedMetadata, meta);
-        absorbTiming(trackedTiming, evalItem);
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
-        }
-      }
-    } catch (err) {
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.error = err instanceof Error ? err.message : String(err);
-      if (options?.captureTraces) {
-        const captured = err.axlCapturedTraces;
-        if (Array.isArray(captured) && captured.length > 0) {
-          evalItem.traces = captured;
-        }
-      }
-      evalItems[itemIndex] = evalItem;
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    if (budgetLimit != null && totalCost > budgetLimit) {
-      budgetExceeded = true;
-    }
-    const itemScorerCost = await scoreItem(
-      evalItem,
-      config.scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
-    );
-    totalCost += itemScorerCost;
-    evalItems[itemIndex] = evalItem;
-    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-  }
-  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
-  const failures = evalItems.filter((i) => i.error).length;
-  const scorerNames = config.scorers.map((s) => s.name);
-  const scorerStats = {};
-  for (const name of scorerNames) {
-    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    const { scored, failed, skipped } = scorerCounts(evalItems, name);
-    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
-  }
-  const scorerTypes = {};
-  for (const s of config.scorers) {
-    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
-  }
-  let degraded;
-  const rawLimit = config.failOnScorerErrorRate;
-  if (rawLimit != null) {
-    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
-      console.warn(
-        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
-      );
-    } else {
-      for (const name of scorerNames) {
-        const stats = scorerStats[name];
-        const scored = stats.scored ?? 0;
-        const failed = stats.failed ?? 0;
-        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
-        const limit = type === "deterministic" ? 0 : rawLimit;
-        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
-        if (verdict.exceeds) {
-          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
-        }
-      }
-    }
-  }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
-  const modelTiming = runCallSamples.size > 0 ? Object.fromEntries(
-    [...runCallSamples.entries()].map(([model, samples]) => {
-      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
-      const stats = {
-        calls: samples.length,
-        wireMs: computeStats(samples.map((s) => s.wireMs)),
-        queuedMs: computeStats(samples.map((s) => s.queuedMs)),
-        retryMs: computeStats(samples.map((s) => s.retryMs)),
-        ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
-      };
-      return [model, stats];
-    })
-  ) : void 0;
-  const totalModelCalls = /* @__PURE__ */ new Map();
-  for (const item of evalItems) {
-    const itemCounts = item.metadata?.modelCallCounts;
-    if (itemCounts && typeof itemCounts === "object") {
-      for (const [m, count] of Object.entries(itemCounts)) {
-        if (typeof count === "number")
-          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
-      }
-    } else {
-      const itemModels = item.metadata?.models;
-      if (Array.isArray(itemModels)) {
-        for (const m of itemModels) {
-          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
-        }
-      }
-    }
-  }
-  const modelsMeta = {};
-  if (totalModelCalls.size > 0) {
-    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
-    modelsMeta.models = sorted.map(([m]) => m);
-    modelsMeta.modelCounts = Object.fromEntries(sorted);
-  }
-  const totalWorkflowCalls = /* @__PURE__ */ new Map();
-  for (const item of evalItems) {
-    const itemCounts = item.metadata?.workflowCallCounts;
-    if (itemCounts && typeof itemCounts === "object") {
-      for (const [w, count] of Object.entries(itemCounts)) {
-        if (typeof count === "number")
-          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
-      }
-    } else {
-      const itemWorkflows = item.metadata?.workflows;
-      if (Array.isArray(itemWorkflows)) {
-        for (const w of itemWorkflows) {
-          if (typeof w === "string")
-            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
-        }
-      }
-    }
-  }
-  const workflowsMeta = {};
-  if (totalWorkflowCalls.size > 0) {
-    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
-    workflowsMeta.workflows = sorted.map(([w]) => w);
-    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
-  } else if (config.workflow) {
-    workflowsMeta.workflows = [config.workflow];
-    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
-  }
-  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
-  return {
-    id,
-    dataset: config.dataset.name,
-    metadata: {
-      ...config.metadata,
-      scorerTypes,
-      ...modelsMeta,
-      ...workflowsMeta,
-      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
-    },
-    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
-    ...evalItems.some((item) => item.unpriced) ? { unpriced: true } : {},
-    duration: Date.now() - startTime,
-    items: evalItems,
-    summary: {
-      count: items.length,
-      failures,
-      scorers: scorerStats,
-      timing,
-      ...modelTiming ? { modelTiming } : {},
-      ...degraded ? { degraded } : {}
-    }
-  };
-}
-
-// src/rescore.ts
-import { randomUUID as randomUUID2 } from "crypto";
-async function rescore(result, scorers, runtime, options) {
-  const startTime = Date.now();
-  const concurrency = options?.concurrency ?? 5;
-  const scorerConcurrency = options?.scorerConcurrency ?? 5;
-  const scorerContext = {
-    resolveProvider: (uri) => {
-      if (typeof runtime.resolveProvider !== "function") {
-        throw new Error(
-          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
-        );
-      }
-      return runtime.resolveProvider(uri);
-    },
-    signal: options?.signal
-  };
-  const rescored = new Array(result.items.length);
-  let totalCost = 0;
-  async function rescoreItem(original, itemIndex) {
-    if (options?.signal?.aborted) {
-      rescored[itemIndex] = {
-        input: original.input,
-        annotations: original.annotations,
-        output: original.output,
-        error: "cancelled",
-        scores: {}
-      };
-      return;
-    }
-    if (original.error) {
-      rescored[itemIndex] = {
-        input: original.input,
-        annotations: original.annotations,
-        output: original.output,
-        error: original.error,
-        scores: {}
-      };
-      return;
-    }
-    const item = {
-      input: original.input,
-      annotations: original.annotations,
-      output: original.output,
-      metadata: original.metadata,
-      traces: original.traces,
-      scores: {},
-      scoreDetails: {}
-    };
-    const itemScorerCost = await scoreItem(
-      item,
-      scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
-    );
-    totalCost += itemScorerCost;
-    rescored[itemIndex] = item;
-  }
-  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
-  const failures = rescored.filter((i) => i.error).length;
-  const scorerNames = scorers.map((s) => s.name);
-  const scorerStats = {};
-  for (const name of scorerNames) {
-    const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    const { scored, failed, skipped } = scorerCounts(rescored, name);
-    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
-  }
-  const scorerTypes = {};
-  for (const s of scorers) {
-    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
-  }
-  return {
-    id: randomUUID2(),
-    dataset: result.dataset,
-    metadata: (() => {
-      const rest = { ...result.metadata };
-      delete rest.runGroupId;
-      delete rest.runIndex;
-      delete rest.scorerFiltered;
-      delete rest.scorersRun;
-      const merged = {
-        ...rest,
-        rescored: true,
-        originalId: result.id,
-        scorerTypes
-      };
-      if (!Array.isArray(merged.workflows)) {
-        const legacyWorkflow = result.workflow;
-        if (typeof legacyWorkflow === "string" && legacyWorkflow) {
-          merged.workflows = [legacyWorkflow];
-          merged.workflowCounts = { [legacyWorkflow]: result.items.length };
-        }
-      }
-      return merged;
-    })(),
-    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
-    duration: Date.now() - startTime,
-    items: rescored,
-    summary: { count: result.items.length, failures, scorers: scorerStats }
-  };
-}
-
-// src/multi-run.ts
-import { randomUUID as randomUUID3 } from "crypto";
-function std(values, mean) {
-  if (values.length <= 1) return 0;
-  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);
-  return Math.sqrt(sumSqDiffs / (values.length - 1));
-}
-function aggregateRuns(runs) {
-  if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
-  const runGroupId = runs[0].metadata.runGroupId ?? randomUUID3();
-  const { dataset } = runs[0];
-  const seenWorkflows = /* @__PURE__ */ new Set();
-  const workflows = [];
-  for (const run of runs) {
-    const list = run.metadata.workflows;
-    if (Array.isArray(list)) {
-      for (const w of list) {
-        if (typeof w === "string" && !seenWorkflows.has(w)) {
-          seenWorkflows.add(w);
-          workflows.push(w);
-        }
-      }
-    }
-  }
-  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
-  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
-  const scorerNames = Object.keys(runs[0].summary.scorers);
-  const scorers = {};
-  for (const name of scorerNames) {
-    const contributing = runs.filter((r) => {
-      const sc = r.summary.scorers[name]?.scored;
-      return sc == null || sc > 0;
-    });
-    const meanSource = contributing.length > 0 ? contributing : runs;
-    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);
-    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
-    const minMean = Math.min(...means);
-    const maxMean = Math.max(...means);
-    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
-    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
-    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);
-    scorers[name] = {
-      mean: round2(meanOfMeans),
-      std: round2(std(means, meanOfMeans)),
-      min: round2(minMean),
-      max: round2(maxMean),
-      scored,
-      failed,
-      skipped
-    };
-  }
-  let timing;
-  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
-  if (timingMeans.length === runs.length && timingMeans.length > 0) {
-    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
-    timing = {
-      mean: round2(meanTiming),
-      std: round2(std(timingMeans, meanTiming))
-    };
-  }
-  return {
-    runGroupId,
-    runCount: runs.length,
-    workflows,
-    dataset,
-    totalCost: round2(totalCost),
-    totalDuration: round2(totalDuration),
-    scorers,
-    timing
-  };
-}
-
-export {
-  pairedBootstrapCI,
-  scorerCounts,
-  evaluateScorerTolerance,
-  evalCompare,
-  evaluateScorerErrorRateGate,
-  normalizeScorerResult,
-  attachScorerErrorCost,
-  scorer,
-  runEval,
-  rescore,
-  aggregateRuns
-};
-//# sourceMappingURL=chunk-AFUNZE7W.js.map