@tangle-network/agent-eval 0.110.1 → 0.111.0

package/dist/index.js

@@ -9,12 +9,12 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-QFGTU7MT.js";
+} from "./chunk-BOLYPSAE.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
-} from "./chunk-T6W5ADLG.js";
+} from "./chunk-VLNGPUJJ.js";
 import {
   DEFAULT_RULES,
   buildTrajectory,
@@ -23,7 +23,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-PLOMR3HP.js";
+} from "./chunk-YWGKQARF.js";
 import {
   analyzeSeries
 } from "./chunk-BOD4O7OF.js";
@@ -59,7 +59,7 @@ import {
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-GBI5J5DB.js";
+} from "./chunk-D2XDWRFX.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -83,7 +83,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-J4AKLZEV.js";
+} from "./chunk-RCXMWGRY.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -102,7 +102,7 @@ import {
   fileVerdictCache,
   inMemoryVerdictCache,
   summarizeBackendIntegrity
-} from "./chunk-3PFZBGMR.js";
+} from "./chunk-XNCF3JU3.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
@@ -179,11 +179,11 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-6HFYGEZ3.js";
+} from "./chunk-RT2AFUXM.js";
 import {
   runEvalCampaign
-} from "./chunk-6PL5MGDL.js";
-import "./chunk-N22ZO7FV.js";
+} from "./chunk-WZOIFFG2.js";
+import "./chunk-7QPK6W7R.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -194,7 +194,7 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-6MFFKPZ4.js";
+} from "./chunk-EMKORATZ.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -207,6 +207,7 @@ import {
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
   eProcess,
+  holm,
   interRaterReliability,
   interpretCliffs,
   mannWhitneyU,
@@ -232,7 +233,7 @@ import {
   weightedMean,
   wilcoxonSignedRank,
   wilson
-} from "./chunk-XMSYF4A7.js";
+} from "./chunk-VC43KQCK.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -1300,6 +1301,412 @@ async function runE2EWorkflow(client, name, workflow) {
   };
 }
 
+// src/paired-arms.ts
+function pairArms(rows, opts) {
+  const { baselineArm, treatmentArm } = opts;
+  if (baselineArm === treatmentArm) {
+    throw new ValidationError(
+      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
+    );
+  }
+  const byArm = /* @__PURE__ */ new Map();
+  const armsSeen = /* @__PURE__ */ new Set();
+  for (const row of rows) {
+    armsSeen.add(row.arm);
+    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
+    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
+    const group = byKey.get(row.pairKey) ?? [];
+    group.push(row);
+    byKey.set(row.pairKey, group);
+    byArm.set(row.arm, byKey);
+  }
+  for (const arm of [baselineArm, treatmentArm]) {
+    if (!byArm.has(arm)) {
+      const seen = [...armsSeen].sort().join(", ") || "<none>";
+      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
+    }
+  }
+  const baselineByKey = byArm.get(baselineArm);
+  const treatmentByKey = byArm.get(treatmentArm);
+  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
+  const pairs = [];
+  const unpairedBaseline = [];
+  const unpairedTreatment = [];
+  for (const pairKey2 of allKeys) {
+    const b = baselineByKey.get(pairKey2) ?? [];
+    const t = treatmentByKey.get(pairKey2) ?? [];
+    if (b.length <= 1 && t.length <= 1) {
+      if (b.length === 1 && t.length === 1) {
+        pairs.push({ pairKey: pairKey2, repIndex: 0, baseline: b[0], treatment: t[0] });
+      } else {
+        unpairedBaseline.push(...b);
+        unpairedTreatment.push(...t);
+      }
+      continue;
+    }
+    const bByRep = indexByRepKey(b, pairKey2, baselineArm);
+    const tByRep = indexByRepKey(t, pairKey2, treatmentArm);
+    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
+    let repIndex = 0;
+    for (const repKey of repKeys) {
+      const baseline = bByRep.get(repKey);
+      const treatment = tByRep.get(repKey);
+      if (baseline !== void 0 && treatment !== void 0) {
+        pairs.push({ pairKey: pairKey2, repIndex: repIndex++, baseline, treatment });
+      } else if (baseline !== void 0) {
+        unpairedBaseline.push(baseline);
+      } else if (treatment !== void 0) {
+        unpairedTreatment.push(treatment);
+      }
+    }
+  }
+  return { pairs, unpairedBaseline, unpairedTreatment };
+}
+function indexByRepKey(group, pairKey2, arm) {
+  const byRep = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    if (row.repKey === void 0) {
+      throw new ValidationError(
+        `pairArms: pairKey '${pairKey2}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
+      );
+    }
+    if (byRep.has(row.repKey)) {
+      throw new ValidationError(
+        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey2}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
+      );
+    }
+    byRep.set(row.repKey, row);
+  }
+  return byRep;
+}
+function comparePairedArms(rows, opts) {
+  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
+  let correctness = null;
+  const baselinePass = [];
+  const treatmentPass = [];
+  for (const pair of pairs) {
+    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
+    baselinePass.push(pair.baseline.pass ? 1 : 0);
+    treatmentPass.push(pair.treatment.pass ? 1 : 0);
+  }
+  if (baselinePass.length > 0) {
+    const mc = mcnemar(baselinePass, treatmentPass);
+    correctness = {
+      b10: mc.b,
+      b01: mc.c,
+      mcnemar: mc,
+      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
+    };
+  }
+  const metricNames = opts.metricNames ?? [
+    ...new Set(
+      pairs.flatMap((p) => [
+        ...Object.keys(p.baseline.metrics ?? {}),
+        ...Object.keys(p.treatment.metrics ?? {})
+      ])
+    )
+  ].sort();
+  const metricDeltas = metricNames.map((name) => {
+    const before = [];
+    const after = [];
+    let nMissing = 0;
+    for (const pair of pairs) {
+      const b = metricValue(pair.baseline, name);
+      const t = metricValue(pair.treatment, name);
+      if (b === void 0 || t === void 0) {
+        nMissing++;
+        continue;
+      }
+      before.push(b);
+      after.push(t);
+    }
+    const bootstrapCi2 = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
+    return {
+      name,
+      n: before.length,
+      nMissing,
+      medianDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.median,
+      meanDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.mean,
+      bootstrapCi: bootstrapCi2,
+      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
+    };
+  });
+  return {
+    nPairs: pairs.length,
+    nUnpairedBaseline: unpairedBaseline.length,
+    nUnpairedTreatment: unpairedTreatment.length,
+    correctness,
+    metricDeltas
+  };
+}
+function metricValue(row, name) {
+  const v = row.metrics?.[name];
+  if (v === void 0) return void 0;
+  if (!Number.isFinite(v)) {
+    throw new ValidationError(
+      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
+    );
+  }
+  return v;
+}
+
+// src/clustered-paired-binary.ts
+var DEFAULT_BOOTSTRAP_RESAMPLES = 1e4;
+var DEFAULT_SIGN_FLIP_RESAMPLES = 1e5;
+var MAX_RESAMPLES = 1e6;
+var DEFAULT_EXACT_CLUSTER_LIMIT = 20;
+var SIGN_FLIP_SEED_SALT = 2654435769;
+function clusteredPairedBinary(rows, options) {
+  const config = validateOptions(options);
+  const projected = projectSelectedRows(rows, options);
+  const paired = pairArms(projected, {
+    baselineArm: options.baselineArm,
+    treatmentArm: options.treatmentArm
+  });
+  const matchedPairs = paired.pairs.map((pair) => {
+    const baseline = pair.baseline;
+    const treatment = pair.treatment;
+    if (baseline.clusterKey !== treatment.clusterKey) {
+      throw new ValidationError(
+        `clusteredPairedBinary: pairKey '${pair.pairKey}' rep ${pair.repIndex} crosses clusters ('${baseline.clusterKey}' vs '${treatment.clusterKey}')`
+      );
+    }
+    return {
+      pairKey: pair.pairKey,
+      repIndex: pair.repIndex,
+      clusterKey: baseline.clusterKey,
+      baseline: baseline.original,
+      treatment: treatment.original,
+      baselinePass: baseline.pass,
+      treatmentPass: treatment.pass
+    };
+  });
+  const unpairedBaseline = paired.unpairedBaseline.map(
+    (row) => row.original
+  );
+  const unpairedTreatment = paired.unpairedTreatment.map(
+    (row) => row.original
+  );
+  if (matchedPairs.length === 0) {
+    return { matchedPairs, unpairedBaseline, unpairedTreatment, statistics: null };
+  }
+  const clusters = summarizeClusters(matchedPairs);
+  const b10 = clusters.reduce((sum4, cluster) => sum4 + cluster.b10, 0);
+  const b01 = clusters.reduce((sum4, cluster) => sum4 + cluster.b01, 0);
+  const taskWeightedRiskDifference = (b10 - b01) / matchedPairs.length;
+  const equalClusterMean = mean(clusters.map((cluster) => cluster.meanDifference));
+  const bootstrap = clusters.length < 2 ? null : clusterBootstrap(clusters, config);
+  const signFlip = clusterSignFlip(clusters, config);
+  return {
+    matchedPairs,
+    unpairedBaseline,
+    unpairedTreatment,
+    statistics: {
+      nPairs: matchedPairs.length,
+      nClusters: clusters.length,
+      b10,
+      b01,
+      taskWeightedRiskDifference,
+      equalClusterMean,
+      clusters,
+      bootstrap,
+      signFlip
+    }
+  };
+}
+function validateOptions(options) {
+  assertNonEmptyString("baselineArm", options.baselineArm);
+  assertNonEmptyString("treatmentArm", options.treatmentArm);
+  if (options.baselineArm === options.treatmentArm) {
+    throw new ValidationError(
+      `clusteredPairedBinary: baselineArm and treatmentArm are both '${options.baselineArm}'`
+    );
+  }
+  if (!Number.isInteger(options.seed)) {
+    throw new ValidationError(`clusteredPairedBinary: seed must be an integer, got ${options.seed}`);
+  }
+  const confidence = options.confidence ?? 0.95;
+  if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
+    throw new ValidationError(
+      `clusteredPairedBinary: confidence must be in (0,1), got ${confidence}`
+    );
+  }
+  const bootstrapResamples = options.bootstrapResamples ?? DEFAULT_BOOTSTRAP_RESAMPLES;
+  assertResampleCount("bootstrapResamples", bootstrapResamples);
+  const rawMinimumBootstrapResamples = 2 / (1 - confidence);
+  const minimumBootstrapResamples = Math.ceil(
+    rawMinimumBootstrapResamples - Number.EPSILON * Math.max(1, rawMinimumBootstrapResamples) * 8
+  );
+  if (bootstrapResamples < minimumBootstrapResamples) {
+    throw new ValidationError(
+      `clusteredPairedBinary: bootstrapResamples must be at least ${minimumBootstrapResamples} for confidence ${confidence} so both interval tails are represented, got ${bootstrapResamples}`
+    );
+  }
+  const signFlipResamples = options.signFlipResamples ?? DEFAULT_SIGN_FLIP_RESAMPLES;
+  assertResampleCount("signFlipResamples", signFlipResamples);
+  const exactClusterLimit = options.exactClusterLimit ?? DEFAULT_EXACT_CLUSTER_LIMIT;
+  if (!Number.isInteger(exactClusterLimit) || exactClusterLimit < 0 || exactClusterLimit > DEFAULT_EXACT_CLUSTER_LIMIT) {
+    throw new ValidationError(
+      `clusteredPairedBinary: exactClusterLimit must be an integer in [0,${DEFAULT_EXACT_CLUSTER_LIMIT}], got ${exactClusterLimit}`
+    );
+  }
+  const alternative = options.alternative ?? "two-sided";
+  if (alternative !== "two-sided" && alternative !== "greater" && alternative !== "less") {
+    throw new ValidationError(
+      `clusteredPairedBinary: alternative must be 'two-sided', 'greater', or 'less', got ${String(alternative)}`
+    );
+  }
+  return {
+    seed: options.seed,
+    confidence,
+    bootstrapResamples,
+    alternative,
+    exactClusterLimit,
+    signFlipResamples
+  };
+}
+function assertResampleCount(name, value) {
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new ValidationError(
+      `clusteredPairedBinary: ${name} must be a positive integer, got ${value}`
+    );
+  }
+  if (value > MAX_RESAMPLES) {
+    throw new ValidationError(
+      `clusteredPairedBinary: ${name} must not exceed ${MAX_RESAMPLES}, got ${value}`
+    );
+  }
+}
+function projectSelectedRows(rows, options) {
+  const projected = [];
+  for (const original of rows) {
+    const arm = options.arm(original);
+    assertNonEmptyString("arm", arm);
+    if (arm !== options.baselineArm && arm !== options.treatmentArm) continue;
+    const pairKey2 = options.pairKey(original);
+    const clusterKey = options.clusterKey(original);
+    const pass = options.pass(original);
+    const repKey = options.repKey?.(original);
+    assertNonEmptyString("pairKey", pairKey2);
+    assertNonEmptyString("clusterKey", clusterKey);
+    if (typeof pass !== "boolean") {
+      throw new ValidationError(
+        `clusteredPairedBinary: pass accessor must return boolean for pairKey '${pairKey2}'`
+      );
+    }
+    if (repKey !== void 0) assertNonEmptyString("repKey", repKey);
+    projected.push({ pairKey: pairKey2, clusterKey, arm, pass, repKey, original });
+  }
+  return projected;
+}
+function assertNonEmptyString(name, value) {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new ValidationError(
+      `clusteredPairedBinary: ${name} accessor must return a non-empty string`
+    );
+  }
+}
+function summarizeClusters(pairs) {
+  const byCluster = /* @__PURE__ */ new Map();
+  for (const pair of pairs) {
+    const summary = byCluster.get(pair.clusterKey) ?? { nPairs: 0, b10: 0, b01: 0 };
+    summary.nPairs++;
+    if (pair.treatmentPass && !pair.baselinePass) summary.b10++;
+    else if (pair.baselinePass && !pair.treatmentPass) summary.b01++;
+    byCluster.set(pair.clusterKey, summary);
+  }
+  return [...byCluster.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([clusterKey, summary]) => ({
+    clusterKey,
+    ...summary,
+    meanDifference: (summary.b10 - summary.b01) / summary.nPairs
+  }));
+}
+function clusterBootstrap(clusters, config) {
+  const rng = mulberry32(config.seed);
+  const samples = new Array(config.bootstrapResamples);
+  for (let draw = 0; draw < config.bootstrapResamples; draw++) {
+    let differenceSum = 0;
+    let pairCount = 0;
+    for (let index = 0; index < clusters.length; index++) {
+      const cluster = clusters[Math.floor(rng() * clusters.length)];
+      differenceSum += cluster.b10 - cluster.b01;
+      pairCount += cluster.nPairs;
+    }
+    samples[draw] = differenceSum / pairCount;
+  }
+  samples.sort((a, b) => a - b);
+  const alpha = 1 - config.confidence;
+  const lowerIndex = Math.floor(alpha / 2 * config.bootstrapResamples);
+  const upperIndex = Math.min(
+    config.bootstrapResamples - 1,
+    Math.ceil((1 - alpha / 2) * config.bootstrapResamples) - 1
+  );
+  return {
+    statistic: "task-weighted-risk-difference",
+    lower: samples[lowerIndex],
+    upper: samples[Math.max(lowerIndex, upperIndex)],
+    confidence: config.confidence,
+    resamples: config.bootstrapResamples,
+    seed: config.seed
+  };
+}
+function clusterSignFlip(clusters, config) {
+  const clusterTotals = clusters.map((cluster) => cluster.b10 - cluster.b01);
+  const nonZero = clusterTotals.filter((delta) => delta !== 0);
+  const totalPairs = clusters.reduce((sum4, cluster) => sum4 + cluster.nPairs, 0);
+  const statistic = clusterTotals.reduce((sum4, delta) => sum4 + delta, 0) / totalPairs;
+  if (nonZero.length <= config.exactClusterLimit) {
+    const assignments = 2 ** nonZero.length;
+    let extreme2 = 0;
+    for (let mask = 0; mask < assignments; mask++) {
+      let sum4 = 0;
+      for (let index = 0; index < nonZero.length; index++) {
+        sum4 += (mask & 2 ** index ? 1 : -1) * nonZero[index];
+      }
+      const permuted = sum4 / totalPairs;
+      if (isExtreme(permuted, statistic, config.alternative)) extreme2++;
+    }
+    return {
+      statistic,
+      pValue: extreme2 / assignments,
+      alternative: config.alternative,
+      method: "exact",
+      assignments,
+      nClusters: clusters.length,
+      nNonZeroClusters: nonZero.length,
+      seed: null
+    };
+  }
+  const signFlipSeed = (config.seed ^ SIGN_FLIP_SEED_SALT) >>> 0;
+  const rng = mulberry32(signFlipSeed);
+  let extreme = 0;
+  for (let draw = 0; draw < config.signFlipResamples; draw++) {
+    let sum4 = 0;
+    for (const delta of nonZero) sum4 += (rng() < 0.5 ? -1 : 1) * delta;
+    const permuted = sum4 / totalPairs;
+    if (isExtreme(permuted, statistic, config.alternative)) extreme++;
+  }
+  return {
+    statistic,
+    pValue: (extreme + 1) / (config.signFlipResamples + 1),
+    alternative: config.alternative,
+    method: "monte-carlo",
+    assignments: config.signFlipResamples,
+    nClusters: clusters.length,
+    nNonZeroClusters: nonZero.length,
+    seed: signFlipSeed
+  };
+}
+function isExtreme(candidate, observed, alternative) {
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(observed)) * 16;
+  if (alternative === "greater") return candidate >= observed - tolerance;
+  if (alternative === "less") return candidate <= observed + tolerance;
+  return Math.abs(candidate) >= Math.abs(observed) - tolerance;
+}
+function mean(values) {
+  return values.reduce((sum4, value) => sum4 + value, 0) / values.length;
+}
+
 // src/convergence.ts
 var ConvergenceTracker = class {
   criteria;
@@ -2501,155 +2908,6 @@ function liveProofToReleaseTrace(config, trajectory, durationMs) {
   };
 }
 
-// src/paired-arms.ts
-function pairArms(rows, opts) {
-  const { baselineArm, treatmentArm } = opts;
-  if (baselineArm === treatmentArm) {
-    throw new ValidationError(
-      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
-    );
-  }
-  const byArm = /* @__PURE__ */ new Map();
-  const armsSeen = /* @__PURE__ */ new Set();
-  for (const row of rows) {
-    armsSeen.add(row.arm);
-    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-    const group = byKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byKey.set(row.pairKey, group);
-    byArm.set(row.arm, byKey);
-  }
-  for (const arm of [baselineArm, treatmentArm]) {
-    if (!byArm.has(arm)) {
-      const seen = [...armsSeen].sort().join(", ") || "<none>";
-      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
-    }
-  }
-  const baselineByKey = byArm.get(baselineArm);
-  const treatmentByKey = byArm.get(treatmentArm);
-  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-  const pairs = [];
-  const unpairedBaseline = [];
-  const unpairedTreatment = [];
-  for (const pairKey2 of allKeys) {
-    const b = baselineByKey.get(pairKey2) ?? [];
-    const t = treatmentByKey.get(pairKey2) ?? [];
-    if (b.length <= 1 && t.length <= 1) {
-      if (b.length === 1 && t.length === 1) {
-        pairs.push({ pairKey: pairKey2, repIndex: 0, baseline: b[0], treatment: t[0] });
-      } else {
-        unpairedBaseline.push(...b);
-        unpairedTreatment.push(...t);
-      }
-      continue;
-    }
-    const bByRep = indexByRepKey(b, pairKey2, baselineArm);
-    const tByRep = indexByRepKey(t, pairKey2, treatmentArm);
-    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-    let repIndex = 0;
-    for (const repKey of repKeys) {
-      const baseline = bByRep.get(repKey);
-      const treatment = tByRep.get(repKey);
-      if (baseline !== void 0 && treatment !== void 0) {
-        pairs.push({ pairKey: pairKey2, repIndex: repIndex++, baseline, treatment });
-      } else if (baseline !== void 0) {
-        unpairedBaseline.push(baseline);
-      } else if (treatment !== void 0) {
-        unpairedTreatment.push(treatment);
-      }
-    }
-  }
-  return { pairs, unpairedBaseline, unpairedTreatment };
-}
-function indexByRepKey(group, pairKey2, arm) {
-  const byRep = /* @__PURE__ */ new Map();
-  for (const row of group) {
-    if (row.repKey === void 0) {
-      throw new ValidationError(
-        `pairArms: pairKey '${pairKey2}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
-      );
-    }
-    if (byRep.has(row.repKey)) {
-      throw new ValidationError(
-        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey2}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
-      );
-    }
-    byRep.set(row.repKey, row);
-  }
-  return byRep;
-}
-function comparePairedArms(rows, opts) {
-  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-  let correctness = null;
-  const baselinePass = [];
-  const treatmentPass = [];
-  for (const pair of pairs) {
-    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-    baselinePass.push(pair.baseline.pass ? 1 : 0);
-    treatmentPass.push(pair.treatment.pass ? 1 : 0);
-  }
-  if (baselinePass.length > 0) {
-    const mc = mcnemar(baselinePass, treatmentPass);
-    correctness = {
-      b10: mc.b,
-      b01: mc.c,
-      mcnemar: mc,
-      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-    };
-  }
-  const metricNames = opts.metricNames ?? [
-    ...new Set(
-      pairs.flatMap((p) => [
-        ...Object.keys(p.baseline.metrics ?? {}),
-        ...Object.keys(p.treatment.metrics ?? {})
-      ])
-    )
-  ].sort();
-  const metricDeltas = metricNames.map((name) => {
-    const before = [];
-    const after = [];
-    let nMissing = 0;
-    for (const pair of pairs) {
-      const b = metricValue(pair.baseline, name);
-      const t = metricValue(pair.treatment, name);
-      if (b === void 0 || t === void 0) {
-        nMissing++;
-        continue;
-      }
-      before.push(b);
-      after.push(t);
-    }
-    const bootstrapCi2 = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-    return {
-      name,
-      n: before.length,
-      nMissing,
-      medianDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.median,
-      meanDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.mean,
-      bootstrapCi: bootstrapCi2,
-      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-    };
-  });
-  return {
-    nPairs: pairs.length,
-    nUnpairedBaseline: unpairedBaseline.length,
-    nUnpairedTreatment: unpairedTreatment.length,
-    correctness,
-    metricDeltas
-  };
-}
-function metricValue(row, name) {
-  const v = row.metrics?.[name];
-  if (v === void 0) return void 0;
-  if (!Number.isFinite(v)) {
-    throw new ValidationError(
-      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
-    );
-  }
-  return v;
-}
-
 // src/pr-review-benchmark.ts
 var DEFAULT_PR_REVIEW_SCORE_WEIGHTS = {
   recall: 4,
@@ -2714,12 +2972,12 @@ function summarizePrReviewBenchmark(scores2) {
     source,
     caseCount: sourceScores.length,
     commentCount: sum(sourceScores.map((score) => score.commentCount)),
-    aggregateMean: mean(sourceScores.map((score) => score.aggregate)),
-    recallMean: mean(sourceScores.map((score) => score.recall)),
-    precisionMean: mean(sourceScores.map((score) => score.precision)),
-    actionabilityMean: mean(sourceScores.map((score) => score.actionability)),
-    severityCalibrationMean: mean(sourceScores.map((score) => score.severityCalibration)),
-    lowNoiseMean: mean(sourceScores.map((score) => score.lowNoise))
+    aggregateMean: mean2(sourceScores.map((score) => score.aggregate)),
+    recallMean: mean2(sourceScores.map((score) => score.recall)),
+    precisionMean: mean2(sourceScores.map((score) => score.precision)),
+    actionabilityMean: mean2(sourceScores.map((score) => score.actionability)),
+    severityCalibrationMean: mean2(sourceScores.map((score) => score.severityCalibration)),
+    lowNoiseMean: mean2(sourceScores.map((score) => score.lowNoise))
   })).sort((a, b) => b.aggregateMean - a.aggregateMean);
 }
 function aggregatePrReviewScore(dimensions, weights = {}) {
@@ -2815,7 +3073,7 @@ function normalizeTerm(input) {
 function normalizePath(input) {
   return input.replace(/^\.\/+/, "");
 }
-function mean(values) {
+function mean2(values) {
   return values.length ? sum(values) / values.length : 0;
 }
 function sum(values) {
@@ -3605,10 +3863,10 @@ function summarizeHarnessResults(results) {
     return {
       variant,
       runs,
-      aggregateMean: mean2(runs.map((r) => r.aggregate)),
-      passRate: mean2(runs.map((r) => r.score.success)),
-      costUsdMean: mean2(runs.map((r) => r.score.costUsd)),
-      wallSecondsMean: mean2(runs.map((r) => r.score.wallSeconds)),
+      aggregateMean: mean3(runs.map((r) => r.aggregate)),
+      passRate: mean3(runs.map((r) => r.score.success)),
+      costUsdMean: mean3(runs.map((r) => r.score.costUsd)),
+      wallSecondsMean: mean3(runs.map((r) => r.score.wallSeconds)),
       scoreMean: meanRunScore(runs.map((r) => r.score))
     };
   }).sort((a, b) => b.aggregateMean - a.aggregateMean);
@@ -3645,22 +3903,22 @@ async function mapLimit(items, limit, fn) {
   );
   return results;
 }
-function mean2(values) {
+function mean3(values) {
   return values.length ? values.reduce((sum4, value) => sum4 + value, 0) / values.length : 0;
 }
 function meanRunScore(scores2) {
   return {
-    success: mean2(scores2.map((s) => s.success)),
-    goalProgress: mean2(scores2.map((s) => s.goalProgress)),
-    repoGroundedness: mean2(scores2.map((s) => s.repoGroundedness)),
-    driftPenalty: mean2(scores2.map((s) => s.driftPenalty)),
-    toolUseQuality: mean2(scores2.map((s) => s.toolUseQuality)),
-    patchQuality: mean2(scores2.map((s) => s.patchQuality)),
-    testReality: mean2(scores2.map((s) => s.testReality)),
-    finalGate: mean2(scores2.map((s) => s.finalGate)),
-    reviewerBlockers: mean2(scores2.map((s) => s.reviewerBlockers)),
-    costUsd: mean2(scores2.map((s) => s.costUsd)),
-    wallSeconds: mean2(scores2.map((s) => s.wallSeconds)),
+    success: mean3(scores2.map((s) => s.success)),
+    goalProgress: mean3(scores2.map((s) => s.goalProgress)),
+    repoGroundedness: mean3(scores2.map((s) => s.repoGroundedness)),
+    driftPenalty: mean3(scores2.map((s) => s.driftPenalty)),
+    toolUseQuality: mean3(scores2.map((s) => s.toolUseQuality)),
+    patchQuality: mean3(scores2.map((s) => s.patchQuality)),
+    testReality: mean3(scores2.map((s) => s.testReality)),
+    finalGate: mean3(scores2.map((s) => s.finalGate)),
+    reviewerBlockers: mean3(scores2.map((s) => s.reviewerBlockers)),
+    costUsd: mean3(scores2.map((s) => s.costUsd)),
+    wallSeconds: mean3(scores2.map((s) => s.wallSeconds)),
     notes: scores2.flatMap((s) => s.notes ?? [])
   };
 }
@@ -4865,9 +5123,9 @@ function median(sorted) {
   const mid = Math.floor(n / 2);
   return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function stddev(values, mean6) {
+function stddev(values, mean7) {
   if (values.length < 2) return 0;
-  const variance = values.reduce((acc, v) => acc + (v - mean6) ** 2, 0) / values.length;
+  const variance = values.reduce((acc, v) => acc + (v - mean7) ** 2, 0) / values.length;
   return Math.sqrt(variance);
 }
 function computeExperimentStats(reps, thresholds) {
@@ -4893,15 +5151,15 @@ function computeExperimentStats(reps, thresholds) {
     return r.score;
   });
   const sorted = [...scores2].sort((a, b) => a - b);
-  const mean6 = scores2.reduce((s, v) => s + v, 0) / n;
-  const sd = stddev(scores2, mean6);
+  const mean7 = scores2.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores2, mean7);
   const spread = iqr(scores2);
   const rated = reps.filter((r) => typeof r.passed === "boolean");
   const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
   const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
   return {
     median: median(sorted),
-    mean: mean6,
+    mean: mean7,
     min: sorted[0],
     max: sorted[n - 1],
     iqr: spread,
@@ -5072,7 +5330,7 @@ var ExperimentTracker = class {
 };
 
 // src/leaderboard.ts
-function mean3(xs) {
+function mean4(xs) {
   return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null;
 }
 function leaderboard(records, opts) {
@@ -5099,10 +5357,10 @@ function leaderboard(records, opts) {
       n: rs.length,
       passRate: ci.estimate,
       passRateCi95: [ci.lower, ci.upper],
-      meanCostUsd: mean3(rs.map((r) => r.costUsd)),
-      meanTokensIn: mean3(rs.map((r) => r.tokenUsage.input)),
-      meanTokensOut: mean3(rs.map((r) => r.tokenUsage.output)),
-      meanWallMs: mean3(rs.map((r) => r.wallMs))
+      meanCostUsd: mean4(rs.map((r) => r.costUsd)),
+      meanTokensIn: mean4(rs.map((r) => r.tokenUsage.input)),
+      meanTokensOut: mean4(rs.map((r) => r.tokenUsage.output)),
+      meanWallMs: mean4(rs.map((r) => r.wallMs))
     };
   });
   rows.sort(
@@ -6659,12 +6917,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean6 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance = all.reduce((a, v) => a + (v - mean6) ** 2, 0) / all.length;
+  const mean7 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance = all.reduce((a, v) => a + (v - mean7) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance);
-  const ref = Math.abs(mean6) > 1e-9 ? Math.abs(mean6) : 1;
+  const ref = Math.abs(mean7) > 1e-9 ? Math.abs(mean7) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean6, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean7, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6730,8 +6988,8 @@ async function paraphraseRobustnessScenarios(args) {
         });
         scores2.push(out.score);
       }
-      const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-      deltas[m.name] = mean6 - originalScore;
+      const mean7 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+      deltas[m.name] = mean7 - originalScore;
       paraphrasedAll.push(...scores2);
     }
     const paraphrasedMean = paraphrasedAll.length === 0 ? originalScore : paraphrasedAll.reduce((a, b) => a + b, 0) / paraphrasedAll.length;
@@ -7149,8 +7407,8 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
     const scores2 = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
     if (scores2.length < 3) continue;
-    const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-    const variance = scores2.reduce((a, b) => a + (b - mean6) ** 2, 0) / scores2.length;
+    const mean7 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+    const variance = scores2.reduce((a, b) => a + (b - mean7) ** 2, 0) / scores2.length;
     if (variance > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -8666,7 +8924,7 @@ function decideReferenceReplayPromotion(baseline, candidate, policy = {}) {
       regressions
     };
   }
-  const requiredMeanDelta = mean4(compared.map((item) => item.f1Delta));
+  const requiredMeanDelta = mean5(compared.map((item) => item.f1Delta));
   if (requiredMeanDelta < minF1Delta) {
     return {
       promote: false,
@@ -8902,7 +9160,7 @@ function clamp013(value) {
 function sum2(values) {
   return values.reduce((acc, value) => acc + value, 0);
 }
-function mean4(values) {
+function mean5(values) {
   return values.length ? sum2(values) / values.length : 0;
 }
 function formatPct(value) {
@@ -9356,10 +9614,10 @@ var HeldOutGate = class {
       afterHoldout.push(run.outcome.holdoutScore);
     }
     const productiveRuns = beforeHoldout.length;
-    const candidateSearchMean = mean5(scores(candidate, "searchScore", "search"));
-    const candidateHoldoutMean = mean5(scores(candidate, "holdoutScore", "holdout"));
-    const baselineSearchMean = mean5(scores(baseline, "searchScore", "search"));
-    const baselineHoldoutMean = mean5(scores(baseline, "holdoutScore", "holdout"));
+    const candidateSearchMean = mean6(scores(candidate, "searchScore", "search"));
+    const candidateHoldoutMean = mean6(scores(candidate, "holdoutScore", "holdout"));
+    const baselineSearchMean = mean6(scores(baseline, "searchScore", "search"));
+    const baselineHoldoutMean = mean6(scores(baseline, "holdoutScore", "holdout"));
     const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean);
     const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean);
     const medianCandidateCost = medianFinite(candidate.map((r) => r.costUsd));
@@ -9471,7 +9729,7 @@ function scores(runs, field, splitFilter) {
   }
   return out;
 }
-function mean5(xs) {
+function mean6(xs) {
   if (xs.length === 0) return Number.NaN;
   return xs.reduce((s, x) => s + x, 0) / xs.length;
 }
@@ -9511,9 +9769,9 @@ function passOrthogonality(input) {
       sims.push(cosineSimilarity(vectors[i], vectors[j]));
     }
   }
-  const mean6 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
+  const mean7 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
   return {
-    orthogonality: Math.max(0, Math.min(1, 1 - mean6)),
+    orthogonality: Math.max(0, Math.min(1, 1 - mean7)),
     passCount: passes.length,
     similarities: sims
   };
@@ -11530,6 +11788,7 @@ export {
   classifyFailure,
   classifyTreatment,
   cliffsDelta,
+  clusteredPairedBinary,
   codeExecutionJudge,
   cohensD,
   coherenceJudge,
@@ -11668,6 +11927,7 @@ export {
   hashScenarios,
   hashToUnit,
   hiddenGrade,
+  holm,
   htmlContainsElement,
   httpGithubClient,
   improvementVerdict,