@axlsdk/eval 0.23.3 → 0.24.1

package/dist/chunk-HKEAPM4B.js

@@ -0,0 +1,1799 @@
+// src/bootstrap.ts
+function pairedBootstrapCI(differences, options) {
+  const n = differences.length;
+  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };
+  const nResamples = options?.nResamples ?? 1e3;
+  const alpha = options?.alpha ?? 0.05;
+  const mean = differences.reduce((a, b) => a + b, 0) / n;
+  if (n === 1)
+    return {
+      lower: differences[0],
+      upper: differences[0],
+      mean,
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  if (nResamples <= 0)
+    return {
+      lower: round(mean),
+      upper: round(mean),
+      mean: round(mean),
+      pRegression: mean < 0 ? 1 : 0,
+      pImprovement: mean > 0 ? 1 : 0
+    };
+  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();
+  const resampleMeans = new Array(nResamples);
+  for (let r = 0; r < nResamples; r++) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+      sum += differences[Math.floor(rng() * n)];
+    }
+    resampleMeans[r] = sum / n;
+  }
+  resampleMeans.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * nResamples);
+  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;
+  let belowZero = 0;
+  for (let i = 0; i < nResamples; i++) {
+    if (resampleMeans[i] < 0) belowZero++;
+    else break;
+  }
+  let aboveZero = 0;
+  for (let i = nResamples - 1; i >= 0; i--) {
+    if (resampleMeans[i] > 0) aboveZero++;
+    else break;
+  }
+  return {
+    lower: round(resampleMeans[Math.max(0, lowerIdx)]),
+    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),
+    mean: round(mean),
+    pRegression: round(belowZero / nResamples),
+    pImprovement: round(aboveZero / nResamples)
+  };
+}
+function round(n) {
+  return Math.round(n * 1e4) / 1e4;
+}
+function xorshift32(seed) {
+  let state = seed | 0 || 1;
+  return () => {
+    state ^= state << 13;
+    state ^= state >> 17;
+    state ^= state << 5;
+    return (state >>> 0) / 4294967296;
+  };
+}
+
+// src/utils.ts
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
+}
+function round2(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+function formatPercent(rate) {
+  return `${Number((rate * 100).toFixed(1))}%`;
+}
+var DEFAULT_ITEM_ERROR_RATE_LIMIT = 0.05;
+function isErrorRateLimit(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+function evaluateItemErrorRate(items, count, limit) {
+  const failed = items.failed;
+  const attempted = count - items.cancelled - items.budget_skipped - items.budget_interrupted;
+  const rate = attempted > 0 ? failed / attempted : 0;
+  return { failed, attempted, rate, limit, exceeded: attempted > 0 && rate > limit };
+}
+function scorerCounts(items, name) {
+  let scored = 0;
+  let failed = 0;
+  let skipped = 0;
+  for (const i of items) {
+    if (i.error) continue;
+    const detail = i.scoreDetails?.[name];
+    if (detail?.outcome !== void 0) {
+      if (detail.outcome === "scored") scored++;
+      else if (detail.outcome === "failed") failed++;
+      else if (detail.outcome === "skipped") skipped++;
+      continue;
+    }
+    if (i.scores[name] != null) scored++;
+    else if (detail?.skipped === true) skipped++;
+    else if (detail?.duration != null) failed++;
+  }
+  return { scored, failed, skipped };
+}
+function evaluateScorerTolerance(scored, failed, type, limit) {
+  const attempted = scored + failed;
+  const rate = attempted === 0 ? 0 : failed / attempted;
+  const exceeds = attempted > 0 && (type === "deterministic" ? failed > 0 : rate > limit);
+  return { attempted, rate, exceeds, zeroSample: attempted === 0 };
+}
+async function mapWithConcurrency(items, concurrency, task) {
+  const results = new Array(items.length);
+  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;
+  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);
+  let next = 0;
+  async function runNext() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await task(items[i], i);
+    }
+  }
+  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+  return results;
+}
+
+// src/accounting.ts
+import { AxlError, isAdmissionDeniedError } from "@axlsdk/axl";
+var USAGE_KEYS = [
+  "inputTokens",
+  "outputTokens",
+  "reasoningTokens",
+  "cachedTokens",
+  "cacheWriteTokens",
+  "audioSeconds"
+];
+function zeroUsage() {
+  return {
+    inputTokens: 0,
+    outputTokens: 0,
+    reasoningTokens: 0,
+    cachedTokens: 0,
+    cacheWriteTokens: 0,
+    audioSeconds: 0
+  };
+}
+function emptyAccounting() {
+  return {
+    version: 1,
+    currency: "USD",
+    knownCost: 0,
+    completeness: "complete",
+    reasons: {},
+    usage: zeroUsage(),
+    operations: { total: 0, settled: 0, unknown: 0, denied: 0, byKind: {} },
+    breakdown: { generation: 0, judging: 0, external: 0 },
+    provenance: {}
+  };
+}
+function uninstrumentedAccounting() {
+  return { ...emptyAccounting(), completeness: "incomplete", reasons: { uninstrumented: 1 } };
+}
+function isInstrumented(runtime) {
+  return typeof runtime?.trackOutcome === "function";
+}
+async function trackScope(runtime, fn, options) {
+  if (!isInstrumented(runtime)) {
+    try {
+      return {
+        status: "fulfilled",
+        value: await fn(),
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    } catch (error) {
+      return {
+        status: "rejected",
+        error,
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    }
+  }
+  const outcome = await runtime.trackOutcome(fn, options);
+  return { ...outcome, instrumented: true };
+}
+function parseBudget(budget) {
+  const match = typeof budget === "string" ? budget.match(/^\$?(\d+(?:\.\d+)?)$/) : null;
+  if (!match) {
+    throw new AxlError(
+      "INVALID_BUDGET",
+      `Invalid budget "${String(budget)}": expected a non-negative USD amount like "$1" or "0.50".`
+    );
+  }
+  const limit = Number(match[1]);
+  if (!Number.isFinite(limit)) {
+    throw new AxlError("INVALID_BUDGET", `Invalid budget "${budget}": not a finite amount.`);
+  }
+  return limit;
+}
+function usableCost(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+function isAdmissionDenied(err) {
+  return isAdmissionDeniedError(err);
+}
+function readAccounting(result) {
+  if (result?.accounting) return result.accounting;
+  const rescored = result?.metadata?.rescored === true;
+  return {
+    ...emptyAccounting(),
+    knownCost: usableCost(result?.totalCost),
+    completeness: "unverified",
+    scope: rescored ? "rescore" : "run"
+  };
+}
+function refusedWork(coverage) {
+  if (!coverage) return 0;
+  return refusedIn(coverage.items) + Object.values(coverage.scorers ?? {}).reduce((n, s) => n + refusedIn(s), 0);
+}
+function refusedIn(counts) {
+  return atLeastZero(counts?.budget_skipped) + atLeastZero(counts?.budget_interrupted);
+}
+function atLeastZero(value) {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+function isBudgetStopped(summary) {
+  if (summary.budget?.status !== "closed") return false;
+  return refusedWork(summary.coverage) > 0;
+}
+function addUsage(target, source) {
+  if (!source) return;
+  for (const key of USAGE_KEYS) {
+    const value = source[key];
+    if (typeof value === "number" && Number.isFinite(value)) target[key] += value;
+  }
+}
+function addCounts(target, source) {
+  if (!source) return;
+  for (const [key, count] of Object.entries(source)) {
+    if (typeof count === "number" && Number.isFinite(count)) {
+      target[key] = (target[key] ?? 0) + count;
+    }
+  }
+}
+function aggregateAccounting(inputs) {
+  const out = emptyAccounting();
+  let anyUnverified = false;
+  let anyIncomplete = false;
+  for (const input of inputs) {
+    if (!input) continue;
+    if (input.completeness === "unverified") anyUnverified = true;
+    else if (input.completeness === "incomplete") anyIncomplete = true;
+    out.knownCost += usableCost(input.knownCost);
+    addUsage(out.usage, input.usage);
+    addCounts(out.reasons, input.reasons);
+    addCounts(out.operations.byKind, input.operations?.byKind);
+    addCounts(out.provenance, input.provenance);
+    out.operations.total += input.operations?.total ?? 0;
+    out.operations.settled += input.operations?.settled ?? 0;
+    out.operations.unknown += input.operations?.unknown ?? 0;
+    out.operations.denied += input.operations?.denied ?? 0;
+    out.breakdown.generation += usableCost(input.breakdown?.generation);
+    out.breakdown.judging += usableCost(input.breakdown?.judging);
+    out.breakdown.external += usableCost(input.breakdown?.external);
+  }
+  out.completeness = anyUnverified ? "unverified" : anyIncomplete ? "incomplete" : "complete";
+  return out;
+}
+
+// src/scorer.ts
+function normalizeScorerResult(result) {
+  return typeof result === "number" ? { score: result } : result;
+}
+function attachScorerErrorCost(err, cost) {
+  if (err && typeof err === "object" && typeof cost === "number" && Number.isFinite(cost) && cost >= 0) {
+    err.cost = cost;
+  }
+}
+function extractScorerErrorCost(err) {
+  const v = err && typeof err === "object" ? err.cost : void 0;
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
+}
+function scorer(config) {
+  return {
+    name: config.name,
+    description: config.description,
+    isLlm: false,
+    score: config.score,
+    applies: config.applies
+  };
+}
+
+// src/diagnostics.ts
+var DEFAULT_COPY_MAX_BYTES = 16 * 1024 * 1024;
+function resolveCaptureLimits(option) {
+  if (option === void 0 || option === false) return void 0;
+  if (option === true) return {};
+  return {
+    ...option.maxRecordBytes !== void 0 ? { maxRecordBytes: option.maxRecordBytes } : {},
+    ...option.maxRunBytes !== void 0 ? { maxRunBytes: option.maxRunBytes } : {},
+    ...option.maxQueueBytes !== void 0 ? { maxQueueBytes: option.maxQueueBytes } : {}
+  };
+}
+function toDiagnosticManifest(manifest) {
+  return {
+    version: 1,
+    artifactId: manifest.artifactId,
+    fidelity: "runtime_request",
+    status: manifest.status,
+    ...manifest.reason !== void 0 ? { reason: manifest.reason } : {},
+    records: manifest.records,
+    bytes: manifest.bytes,
+    redaction: manifest.redaction,
+    ...manifest.expiresAt !== void 0 ? { expiresAt: manifest.expiresAt } : {}
+  };
+}
+function unavailableManifest(artifactId, reason) {
+  return {
+    version: 1,
+    artifactId,
+    fidelity: "runtime_request",
+    status: "unavailable",
+    reason,
+    records: 0,
+    bytes: 0,
+    redaction: "none"
+  };
+}
+var DEFAULT_SIDECAR_MAX_BYTES = 16 * 1024 * 1024;
+var DEFAULT_SIDECAR_MAX_RECORD_BYTES = 4 * 1024 * 1024;
+function validateRequestSidecar(text, options) {
+  if (typeof text !== "string") {
+    return { ok: false, reason: "requests sidecar must be a JSONL string" };
+  }
+  const maxBytes = options?.maxBytes ?? DEFAULT_SIDECAR_MAX_BYTES;
+  const maxRecordBytes = options?.maxRecordBytes ?? DEFAULT_SIDECAR_MAX_RECORD_BYTES;
+  const bytes = Buffer.byteLength(text, "utf-8");
+  if (bytes > maxBytes) {
+    return { ok: false, reason: `requests sidecar exceeds ${maxBytes} bytes` };
+  }
+  const lines = text.split("\n").filter((line) => line.trim() !== "");
+  for (let i = 0; i < lines.length; i++) {
+    const lineBytes = Buffer.byteLength(lines[i], "utf-8");
+    if (lineBytes > maxRecordBytes) {
+      return {
+        ok: false,
+        reason: `requests sidecar line ${i + 1} exceeds ${maxRecordBytes} bytes`
+      };
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(lines[i]);
+    } catch {
+      return { ok: false, reason: `requests sidecar line ${i + 1} is not valid JSON` };
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return { ok: false, reason: `requests sidecar line ${i + 1} is not an object` };
+    }
+    const record = parsed;
+    if (record.v !== 1) {
+      return {
+        ok: false,
+        reason: `requests sidecar line ${i + 1} declares unsupported codec version ${String(record.v)}`
+      };
+    }
+    if (typeof record.operationId !== "string" || record.operationId === "") {
+      return { ok: false, reason: `requests sidecar line ${i + 1} has no operationId` };
+    }
+    if (record.phase !== "start" && record.phase !== "attempt" && record.phase !== "end") {
+      return { ok: false, reason: `requests sidecar line ${i + 1} has an unknown phase` };
+    }
+  }
+  return { ok: true, lines, bytes };
+}
+function parseRequestRecords(lines) {
+  return lines.map((line) => JSON.parse(line));
+}
+function serializeRequestRecords(records) {
+  return records.map((record) => JSON.stringify(record)).join("\n") + (records.length ? "\n" : "");
+}
+
+// src/runner.ts
+import { AdmissionController, AxlError as AxlError2, ProviderError, RequestCaptureChannel } from "@axlsdk/axl";
+
+// src/score-item.ts
+function describeInput(input) {
+  let rendered;
+  try {
+    rendered = JSON.stringify(input) ?? String(input);
+  } catch {
+    rendered = "[unserializable input]";
+  }
+  return rendered.length > 200 ? `${rendered.slice(0, 200)}\u2026` : rendered;
+}
+async function scoreItem(item, scorers, context) {
+  const scoreDetails = item.scoreDetails ??= {};
+  for (const s of scorers) {
+    item.scores[s.name] = null;
+    scoreDetails[s.name] = { score: null, outcome: "cancelled" };
+  }
+  const { runtime, scorerContext, signal, admission } = context;
+  const scorerErrorsByName = {};
+  await mapWithConcurrency(scorers, context.scorerConcurrency, async (scorer2) => {
+    const record = (detail, outcome2) => {
+      scoreDetails[scorer2.name] = { ...detail, outcome: outcome2 };
+    };
+    if (signal?.aborted) {
+      record({ score: null }, "cancelled");
+      return;
+    }
+    if (admission?.closed && scorer2.isLlm) {
+      record({ score: null }, "budget_skipped");
+      return;
+    }
+    const scorerStart = Date.now();
+    let applicable = true;
+    try {
+      applicable = !scorer2.applies || scorer2.applies(item.output, item.input, item.annotations);
+    } catch (err) {
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record({ score: null, duration: Date.now() - scorerStart }, "failed");
+      return;
+    }
+    if (!applicable) {
+      record({ score: null, skipped: true }, "skipped");
+      return;
+    }
+    const outcome = await trackScope(
+      runtime,
+      async () => scorer2.score(item.output, item.input, item.annotations, scorerContext),
+      // The scorer stamp is merged OVER the enclosing item's correlation, so a
+      // judge's records keep the case index and add the judge's name -- which is
+      // what lets `scoreDetails[name].diagnostics` point at this judge's own
+      // calls rather than at the whole item's.
+      { purpose: "judging", captureCorrelation: { scorer: scorer2.name } }
+    );
+    const duration = Date.now() - scorerStart;
+    const { accounting } = outcome;
+    const measured = accounting.operations.total > 0 || accounting.operations.denied > 0;
+    const withCost = (detail, callerCost) => ({
+      ...detail,
+      ...measured ? { accounting, cost: accounting.knownCost } : {},
+      ...!measured && !outcome.instrumented && callerCost != null ? { cost: callerCost } : {}
+    });
+    if (outcome.status === "rejected") {
+      const err = outcome.error;
+      if (err?.name === "AbortError") {
+        record(withCost({ score: null, duration }, void 0), "cancelled");
+        return;
+      }
+      if (isAdmissionDenied(err)) {
+        context.onClosure?.("operation");
+        record(withCost({ score: null, duration }, void 0), "budget_interrupted");
+        return;
+      }
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record(withCost({ score: null, duration }, extractScorerErrorCost(err)), "failed");
+      if (admission?.closed) context.onClosure?.("scorer");
+      return;
+    }
+    try {
+      const scorerResult = normalizeScorerResult(outcome.value);
+      const callerCost = typeof scorerResult.cost === "number" && Number.isFinite(scorerResult.cost) && scorerResult.cost >= 0 ? scorerResult.cost : void 0;
+      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${describeInput(item.input)}`;
+        record(
+          withCost({ score: null, metadata: scorerResult.metadata, duration }, callerCost),
+          "failed"
+        );
+      } else {
+        item.scores[scorer2.name] = round2(scorerResult.score);
+        record(
+          withCost(
+            {
+              score: round2(scorerResult.score),
+              metadata: scorerResult.metadata,
+              duration
+            },
+            callerCost
+          ),
+          "scored"
+        );
+      }
+    } catch (err) {
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record(withCost({ score: null, duration }, void 0), "failed");
+    }
+    if (admission?.closed) context.onClosure?.("scorer");
+  });
+  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
+  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
+}
+
+// src/runner.ts
+import { randomUUID } from "crypto";
+var RESERVED_METADATA_KEYS = [
+  "models",
+  "modelCallCounts",
+  "workflows",
+  "workflowCallCounts",
+  "tokens",
+  "agentCalls"
+];
+var ITEM_OUTCOMES = [
+  "completed",
+  "failed",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+var SCORER_OUTCOMES = [
+  "scored",
+  "failed",
+  "skipped",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Measured spend is unaffected.`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+function splitCallerMetadata(user) {
+  if (!user) return {};
+  let reserved;
+  const safe = {};
+  for (const [key, value] of Object.entries(user)) {
+    if (RESERVED_METADATA_KEYS.includes(key)) {
+      (reserved ??= {})[key] = value;
+    } else {
+      safe[key] = value;
+    }
+  }
+  return { reserved, ...Object.keys(safe).length > 0 ? { safe } : {} };
+}
+function isAbortError(err) {
+  return err?.name === "AbortError";
+}
+async function stageCapture(runtime, ownerId, option) {
+  const limits = resolveCaptureLimits(option);
+  if (!limits) return void 0;
+  const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+  return {
+    artifactId: staged.artifactId,
+    channel: new RequestCaptureChannel({
+      sink: staged.sink,
+      ...limits,
+      // Redaction is the RUNTIME's policy, not the caller's: a run cannot opt
+      // out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  };
+}
+async function finishCapture(runtime, capture) {
+  const status = await capture.channel.close();
+  try {
+    return await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      status.status,
+      status.reason,
+      // What the channel ACTUALLY applied. The store sees only scrubbed bytes
+      // and cannot tell, so a manifest without this reports every compliance-
+      // mode artifact as unredacted.
+      status.redaction
+    );
+  } catch {
+    return void 0;
+  }
+}
+async function abandonCapture(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+function attachOperationRefs(items, channel) {
+  const byItem = /* @__PURE__ */ new Map();
+  const byScorer = /* @__PURE__ */ new Map();
+  for (const entry of channel.operations()) {
+    if (entry.caseIndex === void 0) continue;
+    const ref = {
+      operationId: entry.operationId,
+      kind: entry.kind,
+      ...entry.turn !== void 0 ? { turn: entry.turn } : {},
+      ...entry.attempt !== void 0 ? { attempt: entry.attempt } : {},
+      status: entry.status
+    };
+    if (entry.scorer !== void 0) {
+      const forItem = byScorer.get(entry.caseIndex) ?? /* @__PURE__ */ new Map();
+      const list = forItem.get(entry.scorer) ?? [];
+      list.push(ref);
+      forItem.set(entry.scorer, list);
+      byScorer.set(entry.caseIndex, forItem);
+    } else {
+      const list = byItem.get(entry.caseIndex) ?? [];
+      list.push(ref);
+      byItem.set(entry.caseIndex, list);
+    }
+  }
+  for (const [caseIndex, operations] of byItem) {
+    const item = items[caseIndex];
+    if (!item) continue;
+    item.diagnostics = { operations: [...item.diagnostics?.operations ?? [], ...operations] };
+  }
+  for (const [caseIndex, forItem] of byScorer) {
+    for (const [scorer2, operations] of forItem) {
+      const detail = items[caseIndex]?.scoreDetails?.[scorer2];
+      if (detail) detail.diagnostics = { operations };
+    }
+  }
+}
+var MAX_CAUSE_DEPTH = 8;
+function isProviderErrorLike(value) {
+  if (value instanceof ProviderError) return true;
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value;
+  return candidate.code === "PROVIDER_ERROR" && candidate.name === "ProviderError";
+}
+function describeItemFailure(thrown) {
+  let current = thrown;
+  for (let depth = 0; depth <= MAX_CAUSE_DEPTH; depth++) {
+    if (typeof current !== "object" || current === null) break;
+    if (isProviderErrorLike(current)) {
+      const { name: name2, provider, status, retryable, requestId } = current;
+      return {
+        name: typeof name2 === "string" ? name2 : "ProviderError",
+        ...typeof provider === "string" ? { provider } : {},
+        ...typeof status === "number" && Number.isFinite(status) ? { status } : {},
+        ...typeof retryable === "boolean" ? { retryable } : {},
+        ...typeof requestId === "string" ? { requestId } : {}
+      };
+    }
+    current = current.cause;
+  }
+  const name = thrown?.name;
+  return typeof thrown === "object" && thrown !== null && typeof name === "string" && name ? { name } : void 0;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
+  const startTime = Date.now();
+  const id = randomUUID();
+  const itemErrorRateLimit = config.failOnItemErrorRate ?? DEFAULT_ITEM_ERROR_RATE_LIMIT;
+  if (!isErrorRateLimit(itemErrorRateLimit)) {
+    throw new AxlError2(
+      "INVALID_ITEM_ERROR_RATE",
+      `Invalid failOnItemErrorRate (${String(itemErrorRateLimit)}): expected a number in [0, 1]; 1 disables the gate.`
+    );
+  }
+  const admission = config.budget != null ? new AdmissionController({ limit: parseBudget(config.budget) }) : void 0;
+  const capture = await stageCapture(runtime, id, options?.captureRequests);
+  let items;
+  try {
+    items = await config.dataset.getItems();
+  } catch (error) {
+    await abandonCapture(runtime, capture);
+    throw error;
+  }
+  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
+  const concurrency = config.concurrency ?? 5;
+  const scorerConcurrency = config.scorerConcurrency ?? 5;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    },
+    signal: options?.signal
+  };
+  const evalItems = new Array(items.length);
+  const callerReported = { costItems: 0, costTotal: 0, metadataItems: 0 };
+  const runCallSamples = /* @__PURE__ */ new Map();
+  function absorbTiming(rollup, evalItem) {
+    if (!rollup) return;
+    const perItem = {};
+    for (const [model, bucket] of Object.entries(rollup)) {
+      const sums = {
+        calls: bucket.calls,
+        queuedMs: bucket.queuedMs,
+        retryMs: bucket.retryMs,
+        wireMs: bucket.wireMs,
+        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
+      };
+      perItem[model] = sums;
+      const samples = bucket.samples;
+      if (!samples) continue;
+      const pooled = runCallSamples.get(model) ?? [];
+      for (const sample of samples) pooled.push(sample);
+      runCallSamples.set(model, pooled);
+    }
+    evalItem.timing = perItem;
+  }
+  function stub(item, outcome, error) {
+    return {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      error,
+      outcome,
+      accounting: emptyAccounting(),
+      cost: 0,
+      scorerCost: 0,
+      scores: {}
+    };
+  }
+  async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = stub(item, "cancelled", "Cancelled");
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
+    if (admission?.closed) {
+      noteClosure("case");
+      evalItems[itemIndex] = stub(item, "budget_skipped", "Budget exceeded");
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
+    const evalItem = {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      scores: {}
+    };
+    const itemStart = Date.now();
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        const generation = await trackScope(
+          runtime,
+          async () => executeWorkflow(item.input, runtime),
+          {
+            purpose: "generation",
+            captureTraces: options?.captureTraces === true,
+            captureTimingSamples: true
+          }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        absorbTiming(generation.modelTiming, evalItem);
+        if (generation.traces && generation.traces.length > 0) {
+          evalItem.traces = generation.traces;
+        }
+        if (generation.status === "rejected") {
+          const err = generation.error;
+          if (isAdmissionDenied(err)) {
+            noteClosure("operation");
+            evalItem.outcome = "budget_interrupted";
+            evalItem.error = "Budget interrupted";
+          } else if (isAbortError(err)) {
+            evalItem.outcome = "cancelled";
+            evalItem.error = "Cancelled";
+          } else {
+            evalItem.outcome = "failed";
+            const failure = describeItemFailure(err);
+            if (failure) evalItem.failure = failure;
+            evalItem.error = err instanceof Error ? err.message : String(err);
+            if (admission?.closed) noteClosure("case");
+          }
+          return;
+        }
+        const result = generation.value;
+        evalItem.outcome = "completed";
+        evalItem.output = result.output;
+        const callerCost = extractUserCost(result);
+        if (callerCost !== void 0) {
+          (evalItem.callerReport ??= {}).cost = callerCost;
+          callerReported.costItems += 1;
+          callerReported.costTotal += callerCost;
+        }
+        const { reserved, safe } = splitCallerMetadata(extractUserMetadata(result));
+        if (reserved) {
+          (evalItem.callerReport ??= {}).metadata = reserved;
+          callerReported.metadataItems += 1;
+        }
+        if (generation.metadata || safe) {
+          evalItem.metadata = { ...generation.metadata, ...safe };
+        }
+        if (admission?.closed) noteClosure("case");
+        await scoreItem(evalItem, config.scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "generation",
+        // Every record produced anywhere under this item — workflow turns,
+        // nested asks, tool continuations, judges — is stamped with the case
+        // index, which is how the result can point an item at its own requests.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
+      }
+    );
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    evalItem.accounting = itemOutcome.accounting;
+    evalItem.cost = itemOutcome.accounting.breakdown.generation;
+    evalItem.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") evalItem.unpriced = true;
+    evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+  }
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
+    },
+    // The capture channel is declared ONCE, at the run scope: its byte budget
+    // and pending queue are per RUN, and every nested item/scorer scope
+    // inherits it rather than opening a competing budget of its own.
+    { admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture(runtime, capture);
+    throw runOutcome.error;
+  }
+  let diagnostics;
+  if (capture) {
+    try {
+      attachOperationRefs(evalItems, capture.channel);
+    } catch (error) {
+      await abandonCapture(runtime, capture);
+      throw error;
+    }
+    const manifest = await finishCapture(runtime, capture);
+    diagnostics = manifest ? toDiagnosticManifest(manifest) : (
+      // An artifact that was never sealed is never committed, so publishing
+      // its id would point a reader at bytes the sweeper is about to reclaim.
+      unavailableManifest("", "the diagnostic artifact could not be finalized")
+    );
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "run",
+    ...admission ? {
+      budget: {
+        ...admission.snapshot(),
+        ...closedBy ? { closedBy } : {}
+      }
+    } : {},
+    ...callerReported.costItems > 0 || callerReported.metadataItems > 0 ? { callerReported } : {}
+  };
+  const failures = evalItems.filter((i) => i.error).length;
+  const scorerNames = config.scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    const { scored, failed, skipped } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
+  }
+  const coverage = buildCoverage(evalItems, scorerNames);
+  const scorerTypes = {};
+  for (const s of config.scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  let degraded;
+  const rawLimit = config.failOnScorerErrorRate;
+  if (rawLimit != null) {
+    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
+      console.warn(
+        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
+      );
+    } else {
+      for (const name of scorerNames) {
+        const stats = scorerStats[name];
+        const scored = stats.scored ?? 0;
+        const failed = stats.failed ?? 0;
+        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
+        const limit = type === "deterministic" ? 0 : rawLimit;
+        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+        if (verdict.exceeds) {
+          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
+        }
+      }
+    }
+  }
+  const itemErrorRate = evaluateItemErrorRate(coverage.items, items.length, itemErrorRateLimit);
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const modelTiming = buildModelTiming(runCallSamples);
+  const totalModelCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.modelCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [m, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
+      }
+    } else {
+      const itemModels = item.metadata?.models;
+      if (Array.isArray(itemModels)) {
+        for (const m of itemModels) {
+          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const modelsMeta = {};
+  if (totalModelCalls.size > 0) {
+    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
+    modelsMeta.models = sorted.map(([m]) => m);
+    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  }
+  const totalWorkflowCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.workflowCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [w, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
+      }
+    } else {
+      const itemWorkflows = item.metadata?.workflows;
+      if (Array.isArray(itemWorkflows)) {
+        for (const w of itemWorkflows) {
+          if (typeof w === "string")
+            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const workflowsMeta = {};
+  if (totalWorkflowCalls.size > 0) {
+    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
+    workflowsMeta.workflows = sorted.map(([w]) => w);
+    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
+  } else if (config.workflow) {
+    workflowsMeta.workflows = [config.workflow];
+    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
+  }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
+  return {
+    id,
+    dataset: config.dataset.name,
+    metadata: {
+      ...config.metadata,
+      scorerTypes,
+      ...modelsMeta,
+      ...workflowsMeta,
+      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
+    },
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
+    duration: Date.now() - startTime,
+    items: evalItems,
+    summary: {
+      count: items.length,
+      failures,
+      coverage,
+      scorers: scorerStats,
+      timing,
+      ...modelTiming ? { modelTiming } : {},
+      ...degraded ? { degraded } : {},
+      ...itemErrorRate.failed > 0 ? { itemErrorRate } : {}
+    }
+  };
+}
+function buildCoverage(items, scorerNames) {
+  const coverage = {
+    items: Object.fromEntries(ITEM_OUTCOMES.map((o) => [o, 0])),
+    scorers: {}
+  };
+  for (const name of scorerNames) {
+    coverage.scorers[name] = Object.fromEntries(SCORER_OUTCOMES.map((o) => [o, 0]));
+  }
+  for (const item of items) {
+    const outcome = item.outcome ?? (item.error ? "failed" : "completed");
+    if (coverage.items[outcome] !== void 0) coverage.items[outcome] += 1;
+    for (const name of scorerNames) {
+      const detail = item.scoreDetails?.[name];
+      if (!detail?.outcome) continue;
+      const bucket = coverage.scorers[name];
+      if (bucket[detail.outcome] !== void 0) bucket[detail.outcome] += 1;
+    }
+  }
+  return coverage;
+}
+function buildModelTiming(runCallSamples) {
+  if (runCallSamples.size === 0) return void 0;
+  return Object.fromEntries(
+    [...runCallSamples.entries()].map(([model, samples]) => {
+      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
+      return [
+        model,
+        {
+          calls: samples.length,
+          wireMs: computeStats(samples.map((s) => s.wireMs)),
+          queuedMs: computeStats(samples.map((s) => s.queuedMs)),
+          retryMs: computeStats(samples.map((s) => s.retryMs)),
+          // Custom providers may omit the optional field; that call adds 0.
+          rateLimitRetries: samples.reduce((sum, s) => sum + (s.rateLimitRetries ?? 0), 0),
+          ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
+        }
+      ];
+    })
+  );
+}
+
+// src/compare.ts
+function certifyCost(baselineRuns, candidateRuns, scorerNames) {
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const accounting = readAccounting(run);
+      if (accounting.completeness === "unverified") {
+        return `${side} run ${run.id} carries no accounting (unverified) \u2014 its total is not a measured figure.`;
+      }
+      if (accounting.completeness === "incomplete") {
+        const reasons = Object.keys(accounting.reasons).join(", ") || "unknown spend";
+        return `${side} run ${run.id} has incomplete accounting (${reasons}) \u2014 its total is a lower bound.`;
+      }
+    }
+  }
+  const scopes = /* @__PURE__ */ new Map();
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const scope = readAccounting(run).scope;
+      const seen = scopes.get(side);
+      if (seen !== void 0 && seen !== scope) {
+        return `${side} mixes accounting scopes ("${seen}" and "${scope}") \u2014 a rescore total covers judging only and cannot be averaged with a full run.`;
+      }
+      scopes.set(side, scope);
+    }
+  }
+  const baselineScope = scopes.get("baseline");
+  const candidateScope = scopes.get("candidate");
+  if (baselineScope !== candidateScope) {
+    return `accounting scope differs (baseline "${baselineScope}" vs candidate "${candidateScope}") \u2014 a rescore total covers judging only.`;
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    if (bRun.items.length !== cRun.items.length) {
+      return `item counts differ (baseline ${bRun.items.length} vs candidate ${cRun.items.length}) \u2014 the runs did different amounts of work.`;
+    }
+    for (let i = 0; i < bRun.items.length; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      const bRan = !bItem.error;
+      const cRan = !cItem.error;
+      if (bRan !== cRan) {
+        const stopped = bRan ? cItem : bItem;
+        return `case coverage differs at item ${i}: ${bRan ? "candidate" : "baseline"} did not complete it (${stopped.outcome ?? stopped.error}) \u2014 the cheaper side did less work.`;
+      }
+      if (!bRan) continue;
+      for (const name of scorerNames) {
+        const bScored = bItem.scores[name] != null;
+        const cScored = cItem.scores[name] != null;
+        if (bScored !== cScored) {
+          return `scorer coverage differs for "${name}" at item ${i}: only ${bScored ? "baseline" : "candidate"} produced a score.`;
+        }
+      }
+    }
+  }
+  return null;
+}
+var DEFAULT_LLM_THRESHOLD = 0.05;
+var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
+var LEGACY_THRESHOLD = 0.1;
+function resolveThreshold(scorerName, options, metadata) {
+  if (typeof options?.thresholds === "number") return options.thresholds;
+  if (options?.thresholds && typeof options.thresholds === "object") {
+    const perScorer = options.thresholds[scorerName];
+    if (perScorer != null) return perScorer;
+  }
+  const scorerTypes = metadata.scorerTypes;
+  if (scorerTypes && scorerName in scorerTypes) {
+    return scorerTypes[scorerName] === "llm" ? DEFAULT_LLM_THRESHOLD : DEFAULT_DETERMINISTIC_THRESHOLD;
+  }
+  return LEGACY_THRESHOLD;
+}
+function evalCompare(baseline, candidate, options) {
+  const baselineRunsRaw = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRunsRaw = Array.isArray(candidate) ? candidate : [candidate];
+  if (baselineRunsRaw.length === 0 || candidateRunsRaw.length === 0) {
+    throw new Error("Cannot compare empty eval result arrays");
+  }
+  const runCount = Math.min(baselineRunsRaw.length, candidateRunsRaw.length);
+  const baselineRuns = baselineRunsRaw.slice(0, runCount);
+  const candidateRuns = candidateRunsRaw.slice(0, runCount);
+  const baselineRef = baselineRunsRaw[0];
+  const candidateRef = candidateRunsRaw[0];
+  if (baselineRef.dataset !== candidateRef.dataset) {
+    throw new Error(
+      `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
+    );
+  }
+  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();
+  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
+    throw new Error(`Cannot compare evals with different scorers`);
+  }
+  const pairedDiffs = {};
+  for (const name of baselineScorerNames) {
+    pairedDiffs[name] = [];
+  }
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    const minLength = Math.min(bRun.items.length, cRun.items.length);
+    for (let i = 0; i < minLength; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      if (bItem.error || cItem.error) continue;
+      for (const name of baselineScorerNames) {
+        const bScore = bItem.scores[name];
+        const cScore = cItem.scores[name];
+        if (bScore != null && cScore != null) {
+          pairedDiffs[name].push(cScore - bScore);
+        }
+      }
+    }
+  }
+  const baselineMeans = {};
+  const candidateMeans = {};
+  for (const name of baselineScorerNames) {
+    baselineMeans[name] = baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / baselineRuns.length;
+    candidateMeans[name] = candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) / candidateRuns.length;
+  }
+  const scorers = {};
+  for (const name of baselineScorerNames) {
+    const bMeanRaw = baselineMeans[name];
+    const cMeanRaw = candidateMeans[name];
+    const deltaRaw = cMeanRaw - bMeanRaw;
+    const bMean = round2(bMeanRaw);
+    const cMean = round2(cMeanRaw);
+    const delta = round2(deltaRaw);
+    const deltaPercent = bMeanRaw > 0 ? round2(deltaRaw / bMeanRaw * 100) : 0;
+    const threshold = resolveThreshold(name, options, baselineRef.metadata);
+    let baselineScored = 0;
+    let baselineFailed = 0;
+    let baselineSkipped = 0;
+    for (const run of baselineRuns) {
+      const c = scorerCounts(run.items, name);
+      baselineScored += c.scored;
+      baselineFailed += c.failed;
+      baselineSkipped += c.skipped;
+    }
+    let candidateScored = 0;
+    let candidateFailed = 0;
+    let candidateSkipped = 0;
+    for (const run of candidateRuns) {
+      const c = scorerCounts(run.items, name);
+      candidateScored += c.scored;
+      candidateFailed += c.failed;
+      candidateSkipped += c.skipped;
+    }
+    const entry = {
+      baselineMean: bMean,
+      candidateMean: cMean,
+      delta,
+      deltaPercent,
+      baselineScored,
+      baselineFailed,
+      candidateScored,
+      candidateFailed,
+      baselineSkipped,
+      candidateSkipped
+    };
+    const diffs = pairedDiffs[name];
+    entry.n = diffs.length;
+    if (diffs.length >= 2) {
+      const ci = pairedBootstrapCI(diffs);
+      entry.ci = { lower: ci.lower, upper: ci.upper };
+      entry.pRegression = ci.pRegression;
+      entry.pImprovement = ci.pImprovement;
+      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;
+      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;
+    }
+    scorers[name] = entry;
+  }
+  const regressions = [];
+  const improvements = [];
+  const itemCount = Math.min(
+    ...baselineRuns.map((r) => r.items.length),
+    ...candidateRuns.map((r) => r.items.length)
+  );
+  for (let i = 0; i < itemCount; i++) {
+    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);
+    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);
+    if (hasBaselineError || hasCandidateError) continue;
+    for (const name of baselineScorerNames) {
+      const bScores = baselineRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      const cScores = candidateRuns.map((r) => r.items[i]?.scores[name]).filter((s) => s != null);
+      if (bScores.length === 0 || cScores.length === 0) continue;
+      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;
+      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;
+      const delta = round2(cAvg - bAvg);
+      const threshold = resolveThreshold(name, options, baselineRef.metadata);
+      if (delta < -threshold)
+        regressions.push({
+          itemIndex: i,
+          input: baselineRef.items[i]?.input,
+          scorer: name,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
+          delta
+        });
+      else if (delta > threshold)
+        improvements.push({
+          itemIndex: i,
+          input: baselineRef.items[i]?.input,
+          scorer: name,
+          baselineScore: round2(bAvg),
+          candidateScore: round2(cAvg),
+          delta
+        });
+    }
+  }
+  let timing;
+  const baselineTimings = baselineRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  const candidateTimings = candidateRuns.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (baselineTimings.length > 0 && candidateTimings.length > 0) {
+    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;
+    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;
+    const delta = round2(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round2((cMean - bMean) / bMean * 100) : 0;
+    timing = { baselineMean: round2(bMean), candidateMean: round2(cMean), delta, deltaPercent };
+  }
+  let cost;
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / candidateRuns.length;
+  const hasAccounting = [...baselineRuns, ...candidateRuns].some((r) => r.accounting != null);
+  if (baselineAvgCost > 0 || candidateAvgCost > 0 || hasAccounting) {
+    const deltaRaw = candidateAvgCost - baselineAvgCost;
+    const delta = round2(deltaRaw);
+    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : null;
+    const verdict = certifyCost(baselineRuns, candidateRuns, baselineScorerNames);
+    cost = {
+      baselineTotal: round2(baselineAvgCost),
+      candidateTotal: round2(candidateAvgCost),
+      delta,
+      deltaPercent,
+      certified: verdict === null,
+      ...verdict !== null ? { reason: verdict } : {}
+    };
+  }
+  const parts = [];
+  for (const name of baselineScorerNames) {
+    const s = scorers[name];
+    if (Math.abs(s.delta) > 1e-3) {
+      const direction = s.delta > 0 ? "improves" : "regresses";
+      const sign = s.delta > 0 ? "+" : "";
+      let sigLabel = "";
+      if (s.significant === true) sigLabel = " (significant)";
+      else if (s.significant === false) sigLabel = " (not significant)";
+      parts.push(
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`
+      );
+    }
+  }
+  if (timing && Math.abs(timing.deltaPercent) > 1) {
+    const dir = timing.delta > 0 ? "slower" : "faster";
+    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
+  }
+  if (cost?.certified && cost.deltaPercent !== null && Math.abs(cost.deltaPercent) > 1) {
+    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
+    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
+  }
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  const baselinePartial = detectPartial(baselineRunsRaw);
+  const candidatePartial = detectPartial(candidateRunsRaw);
+  return {
+    baseline: {
+      id: baselineRef.id,
+      metadata: baselineRef.metadata,
+      runCount,
+      ...baselinePartial ? { partial: baselinePartial } : {}
+    },
+    candidate: {
+      id: candidateRef.id,
+      metadata: candidateRef.metadata,
+      runCount,
+      ...candidatePartial ? { partial: candidatePartial } : {}
+    },
+    scorers,
+    timing,
+    cost,
+    regressions,
+    improvements,
+    summary: summaryStr
+  };
+}
+function detectPartial(runs) {
+  if (runs.length === 0) return void 0;
+  let attempted;
+  for (const r of runs) {
+    const v = r.metadata?.batchAttempted;
+    if (typeof v === "number" && Number.isFinite(v)) {
+      attempted = v;
+      break;
+    }
+  }
+  if (attempted === void 0) return void 0;
+  if (runs.length >= attempted) return void 0;
+  return { completed: runs.length, attempted };
+}
+function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
+  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
+  for (const name of Object.keys(comparison.scorers)) {
+    const s = comparison.scorers[name];
+    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
+    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
+    for (const side of ["baseline", "candidate"]) {
+      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
+      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
+      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+      if (verdict.zeroSample) {
+        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
+        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
+      }
+      if (verdict.exceeds) {
+        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
+        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
+      }
+    }
+  }
+  return null;
+}
+function compareItemErrorRates(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  if (!isErrorRateLimit(maxItemErrorRate)) {
+    throw new Error(
+      `Invalid maxItemErrorRate (${String(maxItemErrorRate)}): expected a number in [0, 1].`
+    );
+  }
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  const rates = [];
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (let r = 0; r < runCount; r++) {
+      const run = runs[r];
+      const coverage = run.summary.coverage?.items;
+      const legacy = coverage === void 0;
+      const verdict = legacy ? evaluateItemErrorRate(
+        buildCoverage(run.items, []).items,
+        run.items.length,
+        maxItemErrorRate
+      ) : evaluateItemErrorRate(coverage, run.summary.count, maxItemErrorRate);
+      rates.push({ side, runIndex: r, runId: run.id, legacy, ...verdict });
+    }
+  }
+  return rates;
+}
+function describeItemErrorRate(rate, multiRun) {
+  const which = multiRun ? ` run ${rate.runIndex + 1} (${rate.runId})` : ` (${rate.runId})`;
+  const legacy = rate.legacy ? ", derived from the items of a pre-0.24 artifact" : "";
+  return `${rate.side}${which} item error rate ${formatPercent(rate.rate)} (${rate.failed}/${rate.attempted} attempted items failed${legacy})`;
+}
+function evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  const rates = compareItemErrorRates(baseline, candidate, maxItemErrorRate);
+  const multiRun = rates.some((r) => r.runIndex > 0);
+  const over = rates.find((r) => r.exceeded);
+  if (!over) return null;
+  return `coverage: ${describeItemErrorRate(over, multiRun)} exceeds the ${formatPercent(maxItemErrorRate)} limit \u2014 its scores cover only the surviving items. Pass --max-item-error-rate <0..1> to accept a thinned side (1 disables the check).`;
+}
+
+// src/rescore.ts
+import { AdmissionController as AdmissionController2, RequestCaptureChannel as RequestCaptureChannel2 } from "@axlsdk/axl";
+import { randomUUID as randomUUID2 } from "crypto";
+var COPY_SHARE_OF_RUN_BOUND = 0.75;
+function describeFailure(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function isDiagnosticsUnavailable(error) {
+  return error?.code === "DIAGNOSTICS_UNAVAILABLE";
+}
+async function beginCapture(source, ownerId, runtime, options) {
+  const limits = resolveCaptureLimits(options?.captureRequests);
+  if (!limits) return {};
+  const open = (artifactId, sink, carriedBytes) => ({
+    artifactId,
+    channel: new RequestCaptureChannel2({
+      sink,
+      ...limits,
+      carriedBytes,
+      // Redaction is the RUNTIME's policy, not the caller's: a rescore cannot
+      // opt out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  });
+  const sourceId = source.diagnostics?.artifactId;
+  let stagedId;
+  try {
+    if (!sourceId) {
+      const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+      stagedId = staged.artifactId;
+      return { capture: { ...open(staged.artifactId, staged.sink, 0), copied: false } };
+    }
+    const runBound = limits.maxRunBytes ?? DEFAULT_COPY_MAX_BYTES;
+    const maxBytes = Math.max(1, Math.floor(runBound * COPY_SHARE_OF_RUN_BOUND));
+    const copied = await runtime.copyDiagnosticArtifact(
+      sourceId,
+      { kind: "eval", id: ownerId },
+      { maxBytes }
+    );
+    if (!copied) {
+      return {
+        degraded: unavailableManifest(
+          "",
+          "the source run's captured requests are no longer available"
+        )
+      };
+    }
+    stagedId = copied.artifactId;
+    const capture = {
+      ...open(copied.artifactId, copied.sink, copied.bytes),
+      copied: true,
+      carriedRedaction: copied.redaction
+    };
+    if (copied.truncated) {
+      capture.copyTruncated = `the copied source records stopped at ${maxBytes} bytes, the share of the ${runBound} byte run limit reserved for them`;
+    }
+    return { capture };
+  } catch (error) {
+    if (stagedId !== void 0) {
+      await runtime.rollbackDiagnosticArtifact(stagedId).catch(() => void 0);
+    }
+    if (isDiagnosticsUnavailable(error)) throw error;
+    return {
+      degraded: unavailableManifest(
+        "",
+        `the captured requests could not be carried forward: ${describeFailure(error)}`
+      )
+    };
+  }
+}
+async function abandonCapture2(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+async function finishCapture2(runtime, capture) {
+  const status = await capture.channel.close();
+  const reason = [capture.copyTruncated, status.reason].filter(Boolean).join("; ") || void 0;
+  const truncated = capture.copyTruncated !== void 0 || status.status === "truncated";
+  const finalStatus = status.status === "unavailable" ? "unavailable" : truncated ? "truncated" : status.status;
+  const redaction = status.redaction === "applied" && (capture.carriedRedaction ?? "applied") === "applied" ? "applied" : "none";
+  try {
+    const manifest = await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      finalStatus,
+      reason,
+      redaction
+    );
+    return toDiagnosticManifest(manifest);
+  } catch {
+    return unavailableManifest("", "the diagnostic artifact could not be finalized");
+  }
+}
+async function rescore(result, scorers, runtime, options) {
+  const startTime = Date.now();
+  const rescoredId = randomUUID2();
+  const concurrency = options?.concurrency ?? 5;
+  const scorerConcurrency = options?.scorerConcurrency ?? 5;
+  const admission = options?.budget != null ? new AdmissionController2({ limit: parseBudget(options.budget) }) : void 0;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    },
+    signal: options?.signal
+  };
+  const rescored = new Array(result.items.length);
+  const { capture, degraded } = await beginCapture(result, rescoredId, runtime, options);
+  const carryDiagnostics = capture?.copied === true;
+  async function rescoreItem(original, itemIndex) {
+    if (options?.signal?.aborted) {
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: "cancelled",
+        outcome: "cancelled",
+        accounting: emptyAccounting(),
+        scores: {}
+      };
+      return;
+    }
+    if (original.error) {
+      const accounting2 = emptyAccounting();
+      rescored[itemIndex] = {
+        input: original.input,
+        annotations: original.annotations,
+        output: original.output,
+        error: original.error,
+        ...original.outcome ? { outcome: original.outcome } : {},
+        // The failure cause is a fact about the source run too.
+        ...original.failure ? { failure: original.failure } : {},
+        accounting: accounting2,
+        // Same rule as a scored item: an item carrying `accounting` always
+        // carries both compat views, so no reader falls through to the legacy
+        // "no accounting → trust the caller's number" branch and reports the
+        // SOURCE run's spend as this rescore's. A passthrough did no work, so
+        // both are a measured $0; the source's spend lives in the run-level
+        // `accounting.source` provenance and nowhere else.
+        cost: accounting2.breakdown.generation,
+        scorerCost: accounting2.breakdown.judging,
+        scores: {}
+      };
+      return;
+    }
+    const item = {
+      input: original.input,
+      annotations: original.annotations,
+      output: original.output,
+      metadata: original.metadata,
+      traces: original.traces,
+      outcome: "completed",
+      // Carried forward VERBATIM, original operation ids included. The rescore
+      // did not re-run the workflow, so these still describe the calls that
+      // produced this output; reminting the ids would break the only link back
+      // to the run that made them. Only carried when the source records were
+      // actually copied into this rescore's artifact — a dangling reference is
+      // worse than none.
+      ...carryDiagnostics && original.diagnostics ? { diagnostics: original.diagnostics } : {},
+      scores: {},
+      scoreDetails: {}
+    };
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        await scoreItem(item, scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "judging",
+        // Every record produced under this item is stamped with the case index,
+        // which is how a `ScorerDetail` can point at its own judge calls.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
+      }
+    );
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    item.accounting = itemOutcome.accounting;
+    item.cost = itemOutcome.accounting.breakdown.generation;
+    item.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") item.unpriced = true;
+    rescored[itemIndex] = item;
+  }
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
+    },
+    // The channel is declared ONCE, at the run scope: its byte budget and
+    // pending queue are per rescore, and every nested item/scorer scope inherits
+    // it rather than opening a competing budget of its own.
+    { purpose: "judging", admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture2(runtime, capture);
+    throw runOutcome.error;
+  }
+  const sourceAccounting = result.accounting;
+  const generation = sourceAccounting?.scope === "rescore" ? sourceAccounting.source?.generation ?? null : sourceAccounting ?? null;
+  let diagnostics = degraded;
+  if (capture) {
+    try {
+      attachOperationRefs(rescored, capture.channel);
+    } catch (error) {
+      await abandonCapture2(runtime, capture);
+      throw error;
+    }
+    diagnostics = await finishCapture2(runtime, capture);
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "rescore",
+    // `?? null` is load-bearing: a legacy source has no accounting and must read
+    // as "unknown generation", never as a synthesized complete record.
+    source: { runId: result.id, generation },
+    ...admission ? { budget: { ...admission.snapshot(), ...closedBy ? { closedBy } : {} } } : {}
+  };
+  const failures = rescored.filter((i) => i.error).length;
+  const scorerNames = scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = rescored.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    const { scored, failed, skipped } = scorerCounts(rescored, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
+  }
+  const scorerTypes = {};
+  for (const s of scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  return {
+    id: rescoredId,
+    dataset: result.dataset,
+    metadata: (() => {
+      const rest = { ...result.metadata };
+      delete rest.runGroupId;
+      delete rest.runIndex;
+      delete rest.scorerFiltered;
+      delete rest.scorersRun;
+      const merged = {
+        ...rest,
+        rescored: true,
+        originalId: result.id,
+        scorerTypes
+      };
+      if (!Array.isArray(merged.workflows)) {
+        const legacyWorkflow = result.workflow;
+        if (typeof legacyWorkflow === "string" && legacyWorkflow) {
+          merged.workflows = [legacyWorkflow];
+          merged.workflowCounts = { [legacyWorkflow]: result.items.length };
+        }
+      }
+      return merged;
+    })(),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
+    duration: Date.now() - startTime,
+    items: rescored,
+    summary: {
+      count: result.items.length,
+      failures,
+      coverage: buildCoverage(rescored, scorerNames),
+      scorers: scorerStats,
+      // A rescore makes no generation calls, so the source run's wall-clock
+      // and provider latency are still the only measurements of these items.
+      // `modelTiming` cannot be rebuilt from `item.timing` (sums, not per-call
+      // samples): carry both forward.
+      ...result.summary.timing ? { timing: { ...result.summary.timing } } : {},
+      ...result.summary.modelTiming ? { modelTiming: structuredClone(result.summary.modelTiming) } : {}
+    }
+  };
+}
+
+// src/multi-run.ts
+import { randomUUID as randomUUID3 } from "crypto";
+function std(values, mean) {
+  if (values.length <= 1) return 0;
+  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);
+  return Math.sqrt(sumSqDiffs / (values.length - 1));
+}
+function aggregateRuns(runs) {
+  if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
+  const runGroupId = runs[0].metadata.runGroupId ?? randomUUID3();
+  const { dataset } = runs[0];
+  const seenWorkflows = /* @__PURE__ */ new Set();
+  const workflows = [];
+  for (const run of runs) {
+    const list = run.metadata.workflows;
+    if (Array.isArray(list)) {
+      for (const w of list) {
+        if (typeof w === "string" && !seenWorkflows.has(w)) {
+          seenWorkflows.add(w);
+          workflows.push(w);
+        }
+      }
+    }
+  }
+  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
+  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
+  const scorerNames = Object.keys(runs[0].summary.scorers);
+  const scorers = {};
+  for (const name of scorerNames) {
+    const contributing = runs.filter((r) => {
+      const sc = r.summary.scorers[name]?.scored;
+      return sc == null || sc > 0;
+    });
+    const meanSource = contributing.length > 0 ? contributing : runs;
+    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);
+    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;
+    const minMean = Math.min(...means);
+    const maxMean = Math.max(...means);
+    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);
+    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);
+    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);
+    scorers[name] = {
+      mean: round2(meanOfMeans),
+      std: round2(std(means, meanOfMeans)),
+      min: round2(minMean),
+      max: round2(maxMean),
+      scored,
+      failed,
+      skipped
+    };
+  }
+  let timing;
+  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing.mean);
+  if (timingMeans.length === runs.length && timingMeans.length > 0) {
+    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;
+    timing = {
+      mean: round2(meanTiming),
+      std: round2(std(timingMeans, meanTiming))
+    };
+  }
+  return {
+    runGroupId,
+    runCount: runs.length,
+    workflows,
+    dataset,
+    totalCost: round2(totalCost),
+    accounting: aggregateAccounting(runs.map((r) => readAccounting(r))),
+    totalDuration: round2(totalDuration),
+    scorers,
+    timing
+  };
+}
+
+export {
+  pairedBootstrapCI,
+  formatPercent,
+  isErrorRateLimit,
+  scorerCounts,
+  evaluateScorerTolerance,
+  readAccounting,
+  refusedWork,
+  isBudgetStopped,
+  aggregateAccounting,
+  normalizeScorerResult,
+  attachScorerErrorCost,
+  scorer,
+  DEFAULT_COPY_MAX_BYTES,
+  resolveCaptureLimits,
+  toDiagnosticManifest,
+  unavailableManifest,
+  DEFAULT_SIDECAR_MAX_BYTES,
+  validateRequestSidecar,
+  parseRequestRecords,
+  serializeRequestRecords,
+  runEval,
+  evalCompare,
+  evaluateScorerErrorRateGate,
+  compareItemErrorRates,
+  describeItemErrorRate,
+  evaluateItemErrorRateGate,
+  rescore,
+  aggregateRuns
+};
+//# sourceMappingURL=chunk-HKEAPM4B.js.map