@tangle-network/agent-eval 0.105.0 → 0.106.1

package/dist/{chunk-BBQILVBH.js → chunk-2HLM4SJJ.js}

@@ -1,6 +1,6 @@
 import {
   recoverTruncatedJson
-} from "./chunk-ZKIL3VOD.js";
+} from "./chunk-3E5KUXYZ.js";
 import {
   clamp01
 } from "./chunk-LOJ2QVCE.js";
@@ -858,4 +858,4 @@ export {
   agentProfileModelId,
   agentProfileHash
 };
-//# sourceMappingURL=chunk-BBQILVBH.js.map
+//# sourceMappingURL=chunk-2HLM4SJJ.js.map

package/dist/{chunk-ZKIL3VOD.js → chunk-3E5KUXYZ.js}

@@ -5,6 +5,9 @@ import {
 import {
   callLlm
 } from "./chunk-FUCQVFMU.js";
+import {
+  pairedBootstrap
+} from "./chunk-XMSYF4A7.js";
 import {
   DEFAULT_REDACTION_RULES
 } from "./chunk-GGE4NNQT.js";
@@ -1051,38 +1054,158 @@ function parseReflectionResponse(raw, maxProposals) {
   return out;
 }
 
+// src/campaign/gates/statistical-heldout.ts
+var TIE_WARN_FRACTION = 0.4;
+function pairHoldout(candidate, baseline, scenarioIds, select) {
+  const cellValue = (byCell, cellId) => {
+    const scores = byCell.get(cellId);
+    if (!scores) return void 0;
+    const vals = [];
+    for (const s of Object.values(scores)) {
+      const v = select(s);
+      if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
+    }
+    if (vals.length === 0) return void 0;
+    return vals.reduce((a, b) => a + b, 0) / vals.length;
+  };
+  const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
+  const candCells = [...candidate.keys()].filter(inScope).sort();
+  const baseCells = [...baseline.keys()].filter(inScope).sort();
+  if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
+    throw new Error(
+      `pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
+    );
+  }
+  const before = [];
+  const after = [];
+  const cellIds = [];
+  for (const cellId of candCells) {
+    const b = cellValue(baseline, cellId);
+    const a = cellValue(candidate, cellId);
+    if (b === void 0 || a === void 0) continue;
+    before.push(b);
+    after.push(a);
+    cellIds.push(cellId);
+  }
+  return { before, after, cellIds };
+}
+function heldoutSignificance(paired, opts = {}) {
+  const deltaThreshold = opts.deltaThreshold ?? 0;
+  const minProductiveRuns = opts.minProductiveRuns ?? 3;
+  const confidence = opts.confidence ?? 0.95;
+  const resamples = opts.resamples ?? 2e3;
+  const seed = opts.seed ?? 1337;
+  const statistic = opts.statistic ?? "mean";
+  const bootstrap = pairedBootstrap(paired.before, paired.after, {
+    confidence,
+    resamples,
+    statistic,
+    seed
+  });
+  const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
+    confidence,
+    resamples,
+    statistic: "median",
+    seed
+  });
+  const n = paired.before.length;
+  let ties = 0;
+  for (let i = 0; i < n; i += 1) {
+    const after = paired.after[i] ?? 0;
+    const before = paired.before[i] ?? 0;
+    if (Math.abs(after - before) < 1e-9) ties += 1;
+  }
+  const tieFraction = n === 0 ? 0 : ties / n;
+  const fewRuns = n < minProductiveRuns;
+  const significant = !fewRuns && bootstrap.low > deltaThreshold;
+  return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
+}
+function detectScale(values) {
+  return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
+}
+function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
+  const out = [];
+  for (const dim of criticalDimensions) {
+    const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
+    if (paired.before.length === 0) continue;
+    const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
+    const bootstrap = pairedBootstrap(paired.before, paired.after, {
+      confidence: opts.confidence ?? 0.95,
+      resamples: opts.resamples ?? 2e3,
+      statistic: "median",
+      seed: opts.seed ?? 1337
+    });
+    out.push({
+      dimension: dim,
+      bootstrap,
+      regressed: bootstrap.low < -tolerance,
+      tolerance,
+      n: paired.before.length
+    });
+  }
+  return out;
+}
+
 // src/campaign/gates/heldout-gate.ts
 function heldOutGate(options) {
   const deltaThreshold = options.deltaThreshold ?? 0.5;
+  const confidence = options.confidence ?? 0.95;
+  const minProductiveRuns = options.minProductiveRuns ?? 3;
+  const resamples = options.resamples ?? 2e3;
+  const seed = options.bootstrapSeed ?? 1337;
   return {
     name: "heldOutGate",
     async decide(ctx) {
+      if (!ctx.baselineJudgeScores) {
+        throw new Error(
+          "heldOutGate: ctx.baselineJudgeScores is required \u2014 comparing candidate scores against themselves would hide a missing baseline"
+        );
+      }
       const scenarioIds = new Set(options.scenarios.map((s) => s.id));
-      const baseline = meanForScenarios(ctx.baselineJudgeScores ?? ctx.judgeScores, scenarioIds);
-      const candidate = meanForScenarios(ctx.judgeScores, scenarioIds);
-      const delta = candidate - baseline;
-      const passed = delta >= deltaThreshold;
+      const sig = heldoutSignificance(
+        pairHoldout(ctx.judgeScores, ctx.baselineJudgeScores, scenarioIds, (s) => s.composite),
+        {
+          deltaThreshold,
+          confidence,
+          minProductiveRuns,
+          resamples,
+          seed
+        }
+      );
+      const delta = sig.bootstrap.mean;
+      const passed = sig.significant;
+      const tieNote = sig.tieFraction >= TIE_WARN_FRACTION ? `, ${(sig.tieFraction * 100).toFixed(0)}% tied` : "";
+      const ci = `${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]`;
       return {
         decision: passed ? "ship" : "hold",
-        reasons: passed ? [`held-out delta ${delta.toFixed(3)} \u2265 ${deltaThreshold}`] : [`held-out delta ${delta.toFixed(3)} < ${deltaThreshold}`],
+        reasons: passed ? [
+          `held-out mean \u0394 ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`
+        ] : [
+          sig.fewRuns ? `held-out: only ${sig.n} paired runs \u2014 too few to claim significance` : `held-out mean \u0394 ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`
+        ],
         contributingGates: [
-          { name: "heldOutGate", passed, detail: { baseline, candidate, delta, deltaThreshold } }
+          {
+            name: "heldOutGate",
+            passed,
+            detail: {
+              deltaMean: delta,
+              deltaMedianDiagnostic: sig.medianBootstrap.median,
+              tieFraction: sig.tieFraction,
+              ciLow: sig.bootstrap.low,
+              ciHigh: sig.bootstrap.high,
+              confidence: sig.bootstrap.confidence,
+              n: sig.n,
+              deltaThreshold,
+              fewRuns: sig.fewRuns,
+              seed
+            }
+          }
         ],
         delta
       };
     }
   };
 }
-function meanForScenarios(judgeScoresByCell, scenarioIds) {
-  const composites = [];
-  for (const [cellId, scores] of judgeScoresByCell) {
-    const scenarioId = cellId.split(":")[0] ?? "";
-    if (!scenarioIds.has(scenarioId)) continue;
-    const vals = Object.values(scores).map((s) => s.composite);
-    if (vals.length > 0) composites.push(vals.reduce((a, b) => a + b, 0) / vals.length);
-  }
-  return composites.length === 0 ? 0 : composites.reduce((a, b) => a + b, 0) / composites.length;
-}
 
 // src/campaign/auto-pr.ts
 import { execSync } from "child_process";
@@ -2009,6 +2132,11 @@ export {
   buildReflectionPrompt,
   renderAnalystEvidence,
   parseReflectionResponse,
+  TIE_WARN_FRACTION,
+  pairHoldout,
+  heldoutSignificance,
+  detectScale,
+  dimensionRegressions,
   heldOutGate,
   openAutoPr,
   campaignMeanComposite,
@@ -2029,4 +2157,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-ZKIL3VOD.js.map
+//# sourceMappingURL=chunk-3E5KUXYZ.js.map