@tangle-network/agent-eval 0.118.3 → 0.119.1

package/dist/{chunk-QWRLW4CT.js → chunk-PSXJ32SR.js}

@@ -5,11 +5,11 @@ import {
   resolveRunDir,
   runCampaign,
   summarizeBackendIntegrity
-} from "./chunk-EGNRE7VA.js";
+} from "./chunk-PAHNGS65.js";
 import {
   clamp01,
   validatePolicyEditCandidateRecord
-} from "./chunk-VNLWDTDA.js";
+} from "./chunk-S5TT5R3L.js";
 import {
   detectRewardHacking
 } from "./chunk-ARU2PZFM.js";
@@ -21,12 +21,15 @@ import {
   stripFencedJson
 } from "./chunk-NJC7U437.js";
 import {
+  mcnemar,
   pairedBootstrap,
-  weightedComposite
+  pairedRiskDifference,
+  weightedComposite,
+  wilcoxonSignedRank
 } from "./chunk-PJQFMIOX.js";
 import {
   CostLedger
-} from "./chunk-MLHRFWQK.js";
+} from "./chunk-JHOJHHU7.js";
 import {
   DEFAULT_REDACTION_RULES
 } from "./chunk-GGE4NNQT.js";
@@ -292,6 +295,155 @@ async function runJudgeChat(tc, input, judgeName, request) {
   return paid.value;
 }
 
+// src/paired-arms.ts
+function pairArms(rows, opts) {
+  const { baselineArm, treatmentArm } = opts;
+  if (baselineArm === treatmentArm) {
+    throw new ValidationError(
+      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
+    );
+  }
+  const byArm = /* @__PURE__ */ new Map();
+  const armsSeen = /* @__PURE__ */ new Set();
+  for (const row of rows) {
+    armsSeen.add(row.arm);
+    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
+    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
+    const group = byKey.get(row.pairKey) ?? [];
+    group.push(row);
+    byKey.set(row.pairKey, group);
+    byArm.set(row.arm, byKey);
+  }
+  for (const arm of [baselineArm, treatmentArm]) {
+    if (!byArm.has(arm)) {
+      const seen = [...armsSeen].sort().join(", ") || "<none>";
+      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
+    }
+  }
+  const baselineByKey = byArm.get(baselineArm);
+  const treatmentByKey = byArm.get(treatmentArm);
+  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
+  const pairs = [];
+  const unpairedBaseline = [];
+  const unpairedTreatment = [];
+  for (const pairKey of allKeys) {
+    const b = baselineByKey.get(pairKey) ?? [];
+    const t = treatmentByKey.get(pairKey) ?? [];
+    if (b.length <= 1 && t.length <= 1) {
+      if (b.length === 1 && t.length === 1) {
+        pairs.push({ pairKey, repIndex: 0, baseline: b[0], treatment: t[0] });
+      } else {
+        unpairedBaseline.push(...b);
+        unpairedTreatment.push(...t);
+      }
+      continue;
+    }
+    const bByRep = indexByRepKey(b, pairKey, baselineArm);
+    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
+    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
+    let repIndex = 0;
+    for (const repKey of repKeys) {
+      const baseline = bByRep.get(repKey);
+      const treatment = tByRep.get(repKey);
+      if (baseline !== void 0 && treatment !== void 0) {
+        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
+      } else if (baseline !== void 0) {
+        unpairedBaseline.push(baseline);
+      } else if (treatment !== void 0) {
+        unpairedTreatment.push(treatment);
+      }
+    }
+  }
+  return { pairs, unpairedBaseline, unpairedTreatment };
+}
+function indexByRepKey(group, pairKey, arm) {
+  const byRep = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    if (row.repKey === void 0) {
+      throw new ValidationError(
+        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
+      );
+    }
+    if (byRep.has(row.repKey)) {
+      throw new ValidationError(
+        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
+      );
+    }
+    byRep.set(row.repKey, row);
+  }
+  return byRep;
+}
+function comparePairedArms(rows, opts) {
+  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
+  let correctness = null;
+  const baselinePass = [];
+  const treatmentPass = [];
+  for (const pair of pairs) {
+    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
+    baselinePass.push(pair.baseline.pass ? 1 : 0);
+    treatmentPass.push(pair.treatment.pass ? 1 : 0);
+  }
+  if (baselinePass.length > 0) {
+    const mc = mcnemar(baselinePass, treatmentPass);
+    correctness = {
+      b10: mc.b,
+      b01: mc.c,
+      mcnemar: mc,
+      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
+    };
+  }
+  const metricNames = opts.metricNames ?? [
+    ...new Set(
+      pairs.flatMap((p) => [
+        ...Object.keys(p.baseline.metrics ?? {}),
+        ...Object.keys(p.treatment.metrics ?? {})
+      ])
+    )
+  ].sort();
+  const metricDeltas = metricNames.map((name) => {
+    const before = [];
+    const after = [];
+    let nMissing = 0;
+    for (const pair of pairs) {
+      const b = metricValue(pair.baseline, name);
+      const t = metricValue(pair.treatment, name);
+      if (b === void 0 || t === void 0) {
+        nMissing++;
+        continue;
+      }
+      before.push(b);
+      after.push(t);
+    }
+    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
+    return {
+      name,
+      n: before.length,
+      nMissing,
+      medianDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.median,
+      meanDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.mean,
+      bootstrapCi,
+      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
+    };
+  });
+  return {
+    nPairs: pairs.length,
+    nUnpairedBaseline: unpairedBaseline.length,
+    nUnpairedTreatment: unpairedTreatment.length,
+    correctness,
+    metricDeltas
+  };
+}
+function metricValue(row, name) {
+  const v = row.metrics?.[name];
+  if (v === void 0) return void 0;
+  if (!Number.isFinite(v)) {
+    throw new ValidationError(
+      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
+    );
+  }
+  return v;
+}
+
 // src/pareto.ts
 function dominates(a, b, objectives) {
   let strictlyBetter = false;
@@ -2970,7 +3122,7 @@ async function runImprovementLoop(opts) {
   const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
   const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
   const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
-  const { runCampaign: runCampaign2 } = await import("./run-campaign-DWC67KJP.js");
+  const { runCampaign: runCampaign2 } = await import("./run-campaign-OBXSN5WK.js");
   const baselineOnHoldout = await runCampaign2({
     ...opts,
     costLedger,
@@ -3552,6 +3704,8 @@ export {
   adversarialJudge,
   createCustomJudge,
   defaultJudges,
+  pairArms,
+  comparePairedArms,
   recoverTruncatedJson,
   dominates,
   paretoFrontier,
@@ -3610,4 +3764,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-QWRLW4CT.js.map
+//# sourceMappingURL=chunk-PSXJ32SR.js.map