@tangle-network/agent-eval 0.121.0 → 0.122.1

package/dist/{chunk-JN2FCO5W.js → chunk-TDHX3BVA.js}

@@ -3,7 +3,6 @@ import {
   assertCodeSurfaceIdentity,
   campaignBreakdown,
   campaignMeanComposite,
-  comparePairedArms,
   costReceiptFromTCloud,
   defaultProductionGate,
   gepaProposer,
@@ -16,7 +15,7 @@ import {
   runImprovementLoop,
   surfaceContentHash,
   surfaceHash
-} from "./chunk-32BZXMSO.js";
+} from "./chunk-VZ6VKOJT.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,
@@ -60,8 +59,11 @@ import {
 } from "./chunk-NJC7U437.js";
 import {
   eProcess,
+  mcnemar,
   mulberry32,
-  pairedBootstrap
+  pairedBootstrap,
+  pairedRiskDifference,
+  wilcoxonSignedRank
 } from "./chunk-PJQFMIOX.js";
 import {
   CostAccountingIncompleteError,
@@ -568,6 +570,155 @@ function failureModeRecallJudge(opts = {}) {
   };
 }
 
+// src/paired-arms.ts
+function pairArms(rows, opts) {
+  const { baselineArm, treatmentArm } = opts;
+  if (baselineArm === treatmentArm) {
+    throw new ValidationError(
+      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
+    );
+  }
+  const byArm = /* @__PURE__ */ new Map();
+  const armsSeen = /* @__PURE__ */ new Set();
+  for (const row of rows) {
+    armsSeen.add(row.arm);
+    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
+    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
+    const group = byKey.get(row.pairKey) ?? [];
+    group.push(row);
+    byKey.set(row.pairKey, group);
+    byArm.set(row.arm, byKey);
+  }
+  for (const arm of [baselineArm, treatmentArm]) {
+    if (!byArm.has(arm)) {
+      const seen = [...armsSeen].sort().join(", ") || "<none>";
+      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
+    }
+  }
+  const baselineByKey = byArm.get(baselineArm);
+  const treatmentByKey = byArm.get(treatmentArm);
+  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
+  const pairs = [];
+  const unpairedBaseline = [];
+  const unpairedTreatment = [];
+  for (const pairKey of allKeys) {
+    const b = baselineByKey.get(pairKey) ?? [];
+    const t = treatmentByKey.get(pairKey) ?? [];
+    if (b.length <= 1 && t.length <= 1) {
+      if (b.length === 1 && t.length === 1) {
+        pairs.push({ pairKey, repIndex: 0, baseline: b[0], treatment: t[0] });
+      } else {
+        unpairedBaseline.push(...b);
+        unpairedTreatment.push(...t);
+      }
+      continue;
+    }
+    const bByRep = indexByRepKey(b, pairKey, baselineArm);
+    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
+    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
+    let repIndex = 0;
+    for (const repKey of repKeys) {
+      const baseline = bByRep.get(repKey);
+      const treatment = tByRep.get(repKey);
+      if (baseline !== void 0 && treatment !== void 0) {
+        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
+      } else if (baseline !== void 0) {
+        unpairedBaseline.push(baseline);
+      } else if (treatment !== void 0) {
+        unpairedTreatment.push(treatment);
+      }
+    }
+  }
+  return { pairs, unpairedBaseline, unpairedTreatment };
+}
+function indexByRepKey(group, pairKey, arm) {
+  const byRep = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    if (row.repKey === void 0) {
+      throw new ValidationError(
+        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
+      );
+    }
+    if (byRep.has(row.repKey)) {
+      throw new ValidationError(
+        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
+      );
+    }
+    byRep.set(row.repKey, row);
+  }
+  return byRep;
+}
+function comparePairedArms(rows, opts) {
+  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
+  let correctness = null;
+  const baselinePass = [];
+  const treatmentPass = [];
+  for (const pair of pairs) {
+    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
+    baselinePass.push(pair.baseline.pass ? 1 : 0);
+    treatmentPass.push(pair.treatment.pass ? 1 : 0);
+  }
+  if (baselinePass.length > 0) {
+    const mc = mcnemar(baselinePass, treatmentPass);
+    correctness = {
+      b10: mc.b,
+      b01: mc.c,
+      mcnemar: mc,
+      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
+    };
+  }
+  const metricNames = opts.metricNames ?? [
+    ...new Set(
+      pairs.flatMap((p) => [
+        ...Object.keys(p.baseline.metrics ?? {}),
+        ...Object.keys(p.treatment.metrics ?? {})
+      ])
+    )
+  ].sort();
+  const metricDeltas = metricNames.map((name) => {
+    const before = [];
+    const after = [];
+    let nMissing = 0;
+    for (const pair of pairs) {
+      const b = metricValue(pair.baseline, name);
+      const t = metricValue(pair.treatment, name);
+      if (b === void 0 || t === void 0) {
+        nMissing++;
+        continue;
+      }
+      before.push(b);
+      after.push(t);
+    }
+    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
+    return {
+      name,
+      n: before.length,
+      nMissing,
+      medianDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.median,
+      meanDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.mean,
+      bootstrapCi,
+      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
+    };
+  });
+  return {
+    nPairs: pairs.length,
+    nUnpairedBaseline: unpairedBaseline.length,
+    nUnpairedTreatment: unpairedTreatment.length,
+    correctness,
+    metricDeltas
+  };
+}
+function metricValue(row, name) {
+  const v = row.metrics?.[name];
+  if (v === void 0) return void 0;
+  if (!Number.isFinite(v)) {
+    throw new ValidationError(
+      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
+    );
+  }
+  return v;
+}
+
 // src/campaign/cross-surface-context.ts
 function validateCrossSurfaceInput(input) {
   assertNonEmptyUnique(input.taskOrder, "taskOrder");
@@ -7874,6 +8025,8 @@ function resolveWorktreePath(surface, worktreeDir) {
 }
 
 export {
+  pairArms,
+  comparePairedArms,
   completionVerdict,
   verifyCompletion,
   parseCorrectnessResponse,
@@ -7955,4 +8108,4 @@ export {
   verifyCodeSurface,
   resolveWorktreePath
 };
-//# sourceMappingURL=chunk-JN2FCO5W.js.map
+//# sourceMappingURL=chunk-TDHX3BVA.js.map