@tangle-network/agent-eval 0.122.9 → 0.123.1

package/dist/{chunk-DTJ6QUQB.js → chunk-VGRCHJON.js}

@@ -74,8 +74,18 @@ function selfNormalizedImportanceWeighting(trajectories, opts = {}) {
 function doublyRobust(trajectories, opts = {}) {
   const cap = opts.weightCap ?? Infinity;
   const clip = opts.rewardClip ?? { low: 0, high: 1 };
-  if (trajectories.length === 0) return zeroEstimate();
+  if (trajectories.length === 0) {
+    return {
+      ...zeroEstimate(),
+      contributionCounts: { dr: 0, ipsFallback: 0, legacyScalar: 0 }
+    };
+  }
   const contributions = [];
+  const contributionCounts = {
+    dr: 0,
+    ipsFallback: 0,
+    legacyScalar: 0
+  };
   let maxW = 0;
   let sumW = 0;
   let sumW2 = 0;
@@ -85,11 +95,32 @@ function doublyRobust(trajectories, opts = {}) {
     }
     const w = Math.min(cap, t.targetProb / t.behaviorProb);
     const r = clamp(t.reward, clip.low, clip.high);
-    const q = typeof t.qHat === "number" && Number.isFinite(t.qHat) ? clamp(t.qHat, clip.low, clip.high) : null;
-    if (q === null) {
-      contributions.push(w * r);
+    const rawQHatChosen = t.qHatChosen;
+    const rawVHatTarget = t.vHatTarget;
+    const hasQHatChosen = rawQHatChosen !== null && rawQHatChosen !== void 0;
+    const hasVHatTarget = rawVHatTarget !== null && rawVHatTarget !== void 0;
+    if (hasQHatChosen !== hasVHatTarget) {
+      throw new ValidationError(
+        `doublyRobust: qHatChosen and vHatTarget must be supplied together (runId=${t.runId})`
+      );
+    }
+    if (hasQHatChosen && hasVHatTarget) {
+      if (!Number.isFinite(rawQHatChosen) || !Number.isFinite(rawVHatTarget)) {
+        throw new ValidationError(
+          `doublyRobust: qHatChosen and vHatTarget must be finite (runId=${t.runId})`
+        );
+      }
+      const qHatChosen = clamp(rawQHatChosen, clip.low, clip.high);
+      const vHatTarget = clamp(rawVHatTarget, clip.low, clip.high);
+      contributions.push(vHatTarget + w * (r - qHatChosen));
+      contributionCounts.dr += 1;
+    } else if (typeof t.qHat === "number" && Number.isFinite(t.qHat)) {
+      const qHat = clamp(t.qHat, clip.low, clip.high);
+      contributions.push(qHat + w * (r - qHat));
+      contributionCounts.legacyScalar += 1;
     } else {
-      contributions.push(q + w * (r - q));
+      contributions.push(w * r);
+      contributionCounts.ipsFallback += 1;
     }
     if (w > maxW) maxW = w;
     sumW += w;
@@ -104,7 +135,8 @@ function doublyRobust(trajectories, opts = {}) {
     standardError: Math.sqrt(variance / n),
     effectiveSampleSize: effN,
     n,
-    maxImportanceWeight: maxW
+    maxImportanceWeight: maxW,
+    contributionCounts
   };
 }
 function offPolicyEstimateAll(trajectories, opts = {}) {
@@ -128,4 +160,4 @@ export {
   doublyRobust,
   offPolicyEstimateAll
 };
-//# sourceMappingURL=chunk-DTJ6QUQB.js.map
+//# sourceMappingURL=chunk-VGRCHJON.js.map

package/dist/chunk-VGRCHJON.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/rl/off-policy.ts"],"sourcesContent":["/**\n * Off-policy evaluation primitives.\n *\n * Standard inverse-probability-weighted (IPS), self-normalized\n * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the\n * value of a *target* policy given trajectories collected under a\n * *behavior* policy. This is the canonical RL eval task: \"we have last\n * week's runs, we changed the policy — how would the new one do without\n * re-running?\"\n *\n * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan\n * & Joachims 2015 for SNIPS) but the *application* to LLM-agent\n * evaluation needs care:\n *\n *   - The \"policy\" is the (prompt, tool config, model snapshot) triple.\n *     Two policies have the same probability over an action *iff* their\n *     LLM call would emit the same token with the same probability —\n *     which is generally unknowable without the model log-probs.\n *   - For LLM agents, propensity scores must be supplied by the caller\n *     (logged in the trace, recovered from token log-probs, or estimated\n *     via a learned propensity model). We do NOT estimate propensity here.\n *   - Doubly-robust requires two outputs from a Q-function: its prediction\n *     for the logged action and its expectation under the target policy.\n *     Consumers compute these with a tabular estimate, regression fit, or\n *     learned reward model before constructing the trajectories.\n *\n * Bias / variance tradeoffs:\n *   - IPS: unbiased; high variance for small overlap, infinite variance\n *     when target has support outside behavior.\n *   - SNIPS: lower variance, slight bias; usually preferred in practice.\n *   - DR: doubly-robust — unbiased if either propensity OR Q-function is\n *     correct. Lowest practical variance when Q is decent. Use this.\n *\n * Caveat the panel will land: on the LLM-agent setting, propensity scores\n * recovered from token log-probs are noisy, the action space is enormous,\n * and overlap is often poor. These estimators are useful but not magic;\n * complement with `replayCampaign` (exact replay where the request hashes\n * match) for high-confidence answers and OPE for the gap.\n */\n\nimport { ValidationError } from '../errors'\n\nexport interface OffPolicyTrajectory {\n  /** Stable id, for traceability through the dataset. */\n  runId: string\n  /** Reward observed under the behavior policy (the realized outcome). */\n  reward: number\n  /**\n   * Behavior-policy probability of the action that was taken. For LLM\n   * agents this is typically `exp(sum(token_log_probs))` over the chosen\n   * trajectory. Must be in (0, 1].\n   */\n  behaviorProb: number\n  /**\n   * Target-policy probability of the same action. For replay-style\n   * counterfactual evaluation this is what the *new* policy would have\n   * assigned to the *old* trajectory. Must be in [0, 1].\n   */\n  targetProb: number\n  /**\n   * Model-based reward prediction for the action selected by the behavior\n   * policy: `Q_hat(context, loggedAction)`. Supply this together with\n   * `vHatTarget` for contextual-bandit doubly-robust estimation.\n   */\n  qHatChosen?: number | null\n  /**\n   * Expected model-based reward under the target policy:\n   * `sum_action targetPolicy(action | context) * Q_hat(context, action)`.\n   * Supply this together with `qHatChosen`. For an honest evaluation, both\n   * values must come from a model cross-fitted or trained outside this row.\n   */\n  vHatTarget?: number | null\n  /**\n   * @deprecated Use `qHatChosen` and `vHatTarget` together. When the new pair\n   * is absent, this scalar is used as both terms to preserve existing results.\n   * When the new pair is present, this field is ignored.\n   */\n  qHat?: number | null\n}\n\nexport interface OffPolicyContributionCounts {\n  /** Contributions using the contextual-bandit doubly-robust formula. */\n  dr: number\n  /** Contributions using exact IPS because no reward-model estimate was supplied. */\n  ipsFallback: number\n  /** Contributions using the deprecated single-scalar formula. */\n  legacyScalar: number\n}\n\nexport interface OffPolicyEstimate {\n  /** Estimated value of the target policy. */\n  value: number\n  /** Standard error of the estimate. */\n  standardError: number\n  /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */\n  effectiveSampleSize: number\n  /** Number of trajectories used. */\n  n: number\n  /**\n   * Diagnostic: maximum importance weight observed. Large values (>>10x\n   * mean) are a red flag — variance is dominated by a few outliers.\n   */\n  maxImportanceWeight: number\n  /** Populated by `doublyRobust` to expose which formula each row used. */\n  contributionCounts?: OffPolicyContributionCounts\n}\n\nexport interface OffPolicyOptions {\n  /**\n   * Cap importance weights at this value (Ionides 2008 truncated IS) to\n   * trade unbiasedness for variance reduction. Default `Infinity` (no cap).\n   * Set e.g. `10` for stable estimates when the policies are close.\n   */\n  weightCap?: number\n  /** Reward clipping range. Default `[0, 1]`. */\n  rewardClip?: { low: number; high: number }\n}\n\n/**\n * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator\n * of E[reward under target policy]. Variance scales with the spread of\n * target/behavior ratios.\n */\nexport function inverseProbabilityWeighting(\n  trajectories: OffPolicyTrajectory[],\n  opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n  const cap = opts.weightCap ?? Infinity\n  const clip = opts.rewardClip ?? { low: 0, high: 1 }\n\n  if (trajectories.length === 0) {\n    return zeroEstimate()\n  }\n\n  const weights: number[] = []\n  const weightedRewards: number[] = []\n  let maxW = 0\n  for (const t of trajectories) {\n    if (t.behaviorProb <= 0) {\n      throw new ValidationError(\n        `inverseProbabilityWeighting: behaviorProb must be > 0 (runId=${t.runId})`,\n      )\n    }\n    const w = Math.min(cap, t.targetProb / t.behaviorProb)\n    const r = clamp(t.reward, clip.low, clip.high)\n    weights.push(w)\n    weightedRewards.push(w * r)\n    if (w > maxW) maxW = w\n  }\n  const n = weights.length\n  const value = weightedRewards.reduce((s, x) => s + x, 0) / n\n  const variance = weightedRewards.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1)\n  const sumW = weights.reduce((s, w) => s + w, 0)\n  const sumW2 = weights.reduce((s, w) => s + w * w, 0)\n  const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n\n  return {\n    value,\n    standardError: Math.sqrt(variance / n),\n    effectiveSampleSize: effN,\n    n,\n    maxImportanceWeight: maxW,\n  }\n}\n\n/**\n * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at\n * the cost of small bias (vanishing as N grows). The right default for\n * LLM-agent evaluation where overlap is often poor.\n */\nexport function selfNormalizedImportanceWeighting(\n  trajectories: OffPolicyTrajectory[],\n  opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n  const cap = opts.weightCap ?? Infinity\n  const clip = opts.rewardClip ?? { low: 0, high: 1 }\n  if (trajectories.length === 0) return zeroEstimate()\n\n  const weights: number[] = []\n  const rewards: number[] = []\n  let maxW = 0\n  for (const t of trajectories) {\n    if (t.behaviorProb <= 0) {\n      throw new ValidationError(\n        `selfNormalizedImportanceWeighting: behaviorProb must be > 0 (runId=${t.runId})`,\n      )\n    }\n    const w = Math.min(cap, t.targetProb / t.behaviorProb)\n    weights.push(w)\n    rewards.push(clamp(t.reward, clip.low, clip.high))\n    if (w > maxW) maxW = w\n  }\n  const sumW = weights.reduce((s, w) => s + w, 0)\n  const sumWR = weights.reduce((s, w, i) => s + w * rewards[i]!, 0)\n  const value = sumW === 0 ? 0 : sumWR / sumW\n  const sumW2 = weights.reduce((s, w) => s + w * w, 0)\n  const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n  // Influence-function-based SE for SNIPS (Owen 2013, Ch. 9).\n  const phi = weights.map((w, i) => w * (rewards[i]! - value))\n  const variance = phi.reduce((s, x) => s + x * x, 0) / Math.max(1, sumW * sumW)\n  return {\n    value,\n    standardError: Math.sqrt(variance),\n    effectiveSampleSize: effN,\n    n: trajectories.length,\n    maxImportanceWeight: maxW,\n  }\n}\n\n/**\n * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).\n *\n *     V_DR = (1/N) * sum_i [ v_hat_target_i\n *              + (target_prob_i / behavior_prob_i) * (r_i - q_hat_chosen_i) ]\n *\n * Unbiased if EITHER:\n *   - the importance ratios are correct (IPS-style validity), OR\n *   - the Q-hat function is correct (model-based validity).\n *\n * In practice both are imperfect, but the residual bias is the *product*\n * of both errors — much smaller than either alone. This is why DR is the\n * default in production OPE pipelines.\n *\n * `qHatChosen` and `vHatTarget` must be supplied together. Rows with neither\n * use the exact IPS contribution. Deprecated `qHat` rows preserve the scalar\n * formula, and a complete new pair takes precedence when both forms exist.\n * `contributionCounts` makes the mix explicit in the result.\n * Callers must cross-fit the Q-function or train it on independent rows;\n * fitting and evaluating Q on the same outcomes leaks the answer.\n */\nexport function doublyRobust(\n  trajectories: OffPolicyTrajectory[],\n  opts: OffPolicyOptions = {},\n): OffPolicyEstimate {\n  const cap = opts.weightCap ?? Infinity\n  const clip = opts.rewardClip ?? { low: 0, high: 1 }\n  if (trajectories.length === 0) {\n    return {\n      ...zeroEstimate(),\n      contributionCounts: { dr: 0, ipsFallback: 0, legacyScalar: 0 },\n    }\n  }\n\n  const contributions: number[] = []\n  const contributionCounts: OffPolicyContributionCounts = {\n    dr: 0,\n    ipsFallback: 0,\n    legacyScalar: 0,\n  }\n  let maxW = 0\n  let sumW = 0\n  let sumW2 = 0\n  for (const t of trajectories) {\n    if (t.behaviorProb <= 0) {\n      throw new ValidationError(`doublyRobust: behaviorProb must be > 0 (runId=${t.runId})`)\n    }\n    const w = Math.min(cap, t.targetProb / t.behaviorProb)\n    const r = clamp(t.reward, clip.low, clip.high)\n    const rawQHatChosen = t.qHatChosen\n    const rawVHatTarget = t.vHatTarget\n    const hasQHatChosen = rawQHatChosen !== null && rawQHatChosen !== undefined\n    const hasVHatTarget = rawVHatTarget !== null && rawVHatTarget !== undefined\n    if (hasQHatChosen !== hasVHatTarget) {\n      throw new ValidationError(\n        `doublyRobust: qHatChosen and vHatTarget must be supplied together (runId=${t.runId})`,\n      )\n    }\n\n    if (hasQHatChosen && hasVHatTarget) {\n      if (!Number.isFinite(rawQHatChosen) || !Number.isFinite(rawVHatTarget)) {\n        throw new ValidationError(\n          `doublyRobust: qHatChosen and vHatTarget must be finite (runId=${t.runId})`,\n        )\n      }\n      const qHatChosen = clamp(rawQHatChosen, clip.low, clip.high)\n      const vHatTarget = clamp(rawVHatTarget, clip.low, clip.high)\n      contributions.push(vHatTarget + w * (r - qHatChosen))\n      contributionCounts.dr += 1\n    } else if (typeof t.qHat === 'number' && Number.isFinite(t.qHat)) {\n      const qHat = clamp(t.qHat, clip.low, clip.high)\n      contributions.push(qHat + w * (r - qHat))\n      contributionCounts.legacyScalar += 1\n    } else {\n      contributions.push(w * r)\n      contributionCounts.ipsFallback += 1\n    }\n    if (w > maxW) maxW = w\n    sumW += w\n    sumW2 += w * w\n  }\n  const n = contributions.length\n  const value = contributions.reduce((s, x) => s + x, 0) / n\n  const variance = contributions.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1)\n  const effN = sumW === 0 ? 0 : (sumW * sumW) / sumW2\n  return {\n    value,\n    standardError: Math.sqrt(variance / n),\n    effectiveSampleSize: effN,\n    n,\n    maxImportanceWeight: maxW,\n    contributionCounts,\n  }\n}\n\n/**\n * Convenience: run all three estimators and return them side-by-side.\n * The recommended diagnostic — agreement across estimators is a much\n * stronger signal than any single one.\n */\nexport function offPolicyEstimateAll(\n  trajectories: OffPolicyTrajectory[],\n  opts: OffPolicyOptions = {},\n): { ips: OffPolicyEstimate; snips: OffPolicyEstimate; dr: OffPolicyEstimate } {\n  return {\n    ips: inverseProbabilityWeighting(trajectories, opts),\n    snips: selfNormalizedImportanceWeighting(trajectories, opts),\n    dr: doublyRobust(trajectories, opts),\n  }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction zeroEstimate(): OffPolicyEstimate {\n  return { value: 0, standardError: 0, effectiveSampleSize: 0, n: 0, maxImportanceWeight: 0 }\n}\n\nfunction clamp(x: number, lo: number, hi: number): number {\n  if (!Number.isFinite(x)) return lo\n  return Math.max(lo, Math.min(hi, x))\n}\n"],"mappings":";;;;;AA2HO,SAAS,4BACd,cACA,OAAyB,CAAC,GACP;AACnB,QAAM,MAAM,KAAK,aAAa;AAC9B,QAAM,OAAO,KAAK,cAAc,EAAE,KAAK,GAAG,MAAM,EAAE;AAElD,MAAI,aAAa,WAAW,GAAG;AAC7B,WAAO,aAAa;AAAA,EACtB;AAEA,QAAM,UAAoB,CAAC;AAC3B,QAAM,kBAA4B,CAAC;AACnC,MAAI,OAAO;AACX,aAAW,KAAK,cAAc;AAC5B,QAAI,EAAE,gBAAgB,GAAG;AACvB,YAAM,IAAI;AAAA,QACR,gEAAgE,EAAE,KAAK;AAAA,MACzE;AAAA,IACF;AACA,UAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;AACrD,UAAM,IAAI,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI;AAC7C,YAAQ,KAAK,CAAC;AACd,oBAAgB,KAAK,IAAI,CAAC;AAC1B,QAAI,IAAI,KAAM,QAAO;AAAA,EACvB;AACA,QAAM,IAAI,QAAQ;AAClB,QAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAC3D,QAAM,WAAW,gBAAgB,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;AAC9F,QAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC9C,QAAM,QAAQ,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC;AACnD,QAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;AAE9C,SAAO;AAAA,IACL;AAAA,IACA,eAAe,KAAK,KAAK,WAAW,CAAC;AAAA,IACrC,qBAAqB;AAAA,IACrB;AAAA,IACA,qBAAqB;AAAA,EACvB;AACF;AAOO,SAAS,kCACd,cACA,OAAyB,CAAC,GACP;AACnB,QAAM,MAAM,KAAK,aAAa;AAC9B,QAAM,OAAO,KAAK,cAAc,EAAE,KAAK,GAAG,MAAM,EAAE;AAClD,MAAI,aAAa,WAAW,EAAG,QAAO,aAAa;AAEnD,QAAM,UAAoB,CAAC;AAC3B,QAAM,UAAoB,CAAC;AAC3B,MAAI,OAAO;AACX,aAAW,KAAK,cAAc;AAC5B,QAAI,EAAE,gBAAgB,GAAG;AACvB,YAAM,IAAI;AAAA,QACR,sEAAsE,EAAE,KAAK;AAAA,MAC/E;AAAA,IACF;AACA,UAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;AACrD,YAAQ,KAAK,CAAC;AACd,YAAQ,KAAK,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI,CAAC;AACjD,QAAI,IAAI,KAAM,QAAO;AAAA,EACvB;AACA,QAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC9C,QAAM,QAAQ,QAAQ,OAAO,CAAC,GAAG,GAAG,MAAM,IAAI,IAAI,QAAQ,CAAC,GAAI,CAAC;AAChE,QAAM,QAAQ,SAAS,IAAI,IAAI,QAAQ;AACvC,QAAM,QAAQ,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC;AACnD,QAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;AAE9C,QAAM,MAAM,QAAQ,IAAI,CAAC,GAAG,MAAM,KAAK,QAAQ,CAAC,IAAK,MAAM;AAC3D,QAAM,WAAW,IAAI,OAAO,CAAC,GAAG,MAAM,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,OAAO,IAAI;AAC7E,SAAO;AAAA,IACL;AAAA,IACA,eAAe,KAAK,KAAK,QAAQ;AAAA,IACjC,qBAAqB;AAAA,IACrB,GAAG,aAAa;AAAA,IAChB,qBAAqB;AAAA,EACvB;AACF;AAuBO,SAAS,aACd,cACA,OAAyB,CAAC,GACP;AACnB,QAAM,MAAM,KAAK,aAAa;AAC9B,QAAM,OAAO,KAAK,cAAc,EAAE,KAAK,GAAG,MAAM,EAAE;AAClD,MAAI,aAAa,WAAW,GAAG;AAC7B,WAAO;AAAA,MACL,GAAG,aAAa;AAAA,MAChB,oBAAoB,EAAE,IAAI,GAAG,aAAa,GAAG,cAAc,EAAE;AAAA,IAC/D;AAAA,EACF;AAEA,QAAM,gBAA0B,CAAC;AACjC,QAAM,qBAAkD;AAAA,IACtD,IAAI;AAAA,IACJ,aAAa;AAAA,IACb,cAAc;AAAA,EAChB;AACA,MAAI,OAAO;AACX,MAAI,OAAO;AACX,MAAI,QAAQ;AACZ,aAAW,KAAK,cAAc;AAC5B,QAAI,EAAE,gBAAgB,GAAG;AACvB,YAAM,IAAI,gBAAgB,iDAAiD,EAAE,KAAK,GAAG;AAAA,IACvF;AACA,UAAM,IAAI,KAAK,IAAI,KAAK,EAAE,aAAa,EAAE,YAAY;AACrD,UAAM,IAAI,MAAM,EAAE,QAAQ,KAAK,KAAK,KAAK,IAAI;AAC7C,UAAM,gBAAgB,EAAE;AACxB,UAAM,gBAAgB,EAAE;AACxB,UAAM,gBAAgB,kBAAkB,QAAQ,kBAAkB;AAClE,UAAM,gBAAgB,kBAAkB,QAAQ,kBAAkB;AAClE,QAAI,kBAAkB,eAAe;AACnC,YAAM,IAAI;AAAA,QACR,4EAA4E,EAAE,KAAK;AAAA,MACrF;AAAA,IACF;AAEA,QAAI,iBAAiB,eAAe;AAClC,UAAI,CAAC,OAAO,SAAS,aAAa,KAAK,CAAC,OAAO,SAAS,aAAa,GAAG;AACtE,cAAM,IAAI;AAAA,UACR,iEAAiE,EAAE,KAAK;AAAA,QAC1E;AAAA,MACF;AACA,YAAM,aAAa,MAAM,eAAe,KAAK,KAAK,KAAK,IAAI;AAC3D,YAAM,aAAa,MAAM,eAAe,KAAK,KAAK,KAAK,IAAI;AAC3D,oBAAc,KAAK,aAAa,KAAK,IAAI,WAAW;AACpD,yBAAmB,MAAM;AAAA,IAC3B,WAAW,OAAO,EAAE,SAAS,YAAY,OAAO,SAAS,EAAE,IAAI,GAAG;AAChE,YAAM,OAAO,MAAM,EAAE,MAAM,KAAK,KAAK,KAAK,IAAI;AAC9C,oBAAc,KAAK,OAAO,KAAK,IAAI,KAAK;AACxC,yBAAmB,gBAAgB;AAAA,IACrC,OAAO;AACL,oBAAc,KAAK,IAAI,CAAC;AACxB,yBAAmB,eAAe;AAAA,IACpC;AACA,QAAI,IAAI,KAAM,QAAO;AACrB,YAAQ;AACR,aAAS,IAAI;AAAA,EACf;AACA,QAAM,IAAI,cAAc;AACxB,QAAM,QAAQ,cAAc,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AACzD,QAAM,WAAW,cAAc,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;AAC5F,QAAM,OAAO,SAAS,IAAI,IAAK,OAAO,OAAQ;AAC9C,SAAO;AAAA,IACL;AAAA,IACA,eAAe,KAAK,KAAK,WAAW,CAAC;AAAA,IACrC,qBAAqB;AAAA,IACrB;AAAA,IACA,qBAAqB;AAAA,IACrB;AAAA,EACF;AACF;AAOO,SAAS,qBACd,cACA,OAAyB,CAAC,GACmD;AAC7E,SAAO;AAAA,IACL,KAAK,4BAA4B,cAAc,IAAI;AAAA,IACnD,OAAO,kCAAkC,cAAc,IAAI;AAAA,IAC3D,IAAI,aAAa,cAAc,IAAI;AAAA,EACrC;AACF;AAIA,SAAS,eAAkC;AACzC,SAAO,EAAE,OAAO,GAAG,eAAe,GAAG,qBAAqB,GAAG,GAAG,GAAG,qBAAqB,EAAE;AAC5F;AAEA,SAAS,MAAM,GAAW,IAAY,IAAoB;AACxD,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC;AACrC;","names":[]}

package/dist/contract/index.d.ts

@@ -2386,6 +2386,15 @@ type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptim
     /** Holdout scenarios kept OUT of the training optimization pool — used
      *  ONLY to score baseline vs winner for the gate. */
     holdoutScenarios: TScenario[];
+    /** Holdout policy. Default `'measured'`: baseline + winner are re-scored on
+     *  `holdoutScenarios` and the gate decides on that held-out comparison.
+     *  `'deferred'`: the improvement-set (search) campaigns run exactly as usual,
+     *  but ZERO holdout cells are dispatched, the gate is forced to `'hold'`, and
+     *  the result + provenance record carry `holdout: 'deferred'` with NO
+     *  held-out lift — for callers that measure the held-out comparison in a
+     *  separate later run instead of faking a static holdout scenario and
+     *  recording a meaningless lift. */
+    holdout?: 'measured' | 'deferred';
     /** Promotion gate. Substrate strongly recommends `defaultProductionGate`
      *  for production wiring (composes red-team / reward-hacking / canary /
      *  heldout). */
@@ -2413,6 +2422,10 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
     neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario>;
     neutralizedSurface?: MutableSurface;
     gateResult: Awaited<ReturnType<Gate<TArtifact, TScenario>['decide']>>;
+    /** Present iff the loop ran with `holdout: 'deferred'`. When set,
+     *  `baselineOnHoldout`/`winnerOnHoldout` are the shared EMPTY campaign (zero
+     *  cells dispatched) and the gate verdict is the forced `'hold'`. */
+    holdout?: 'deferred';
     /** Unified baseline→winner surface diff. Computed UNCONDITIONALLY (not only
      *  when `autoOnPromote === 'pr'`) so the diff that the gate decided on is
      *  always present on the result + in the emitted provenance record. Empty
@@ -3715,12 +3728,18 @@ interface LoopProvenanceRecord {
             detail: unknown;
         }>;
     };
-    /** baseline-on-holdout composite mean. */
-    baselineHoldoutComposite: number;
-    /** winner-on-holdout composite mean. */
-    winnerHoldoutComposite: number;
-    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. */
-    heldOutLift: number;
+    /** Present iff the loop ran with `holdout: 'deferred'` — the held-out
+     *  comparison was intentionally not measured in this run, so the holdout
+     *  composites and `heldOutLift` are ABSENT rather than recorded as a
+     *  meaningless 0. */
+    holdout?: 'deferred';
+    /** baseline-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    baselineHoldoutComposite?: number;
+    /** winner-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    winnerHoldoutComposite?: number;
+    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. Absent
+     *  when `holdout === 'deferred'` (no held-out measurement ran). */
+    heldOutLift?: number;
     /** Backend provenance: stub-vs-real verdict + worker call count + models. */
     backend: LoopProvenanceBackend;
     totalCostUsd: number;
@@ -3767,8 +3786,22 @@ interface SelfImproveBudget {
     holdoutFraction?: number;
     /** Explicit held-out scenarios; overrides `holdoutFraction`. */
     holdoutScenarios?: Scenario[];
+    /** Holdout policy. Default `'measured'`: split, re-score baseline vs winner
+     *  on the held-out set, gate on that comparison. `'deferred'`: run the
+     *  improvement-set campaigns + search promotion, dispatch ZERO holdout cells,
+     *  force the gate to `'hold'`, return `lift: undefined`, and record
+     *  `holdout: 'deferred'` in the provenance record — for callers that measure
+     *  the held-out comparison in a separate later run instead of faking a
+     *  static holdout scenario and recording a meaningless lift. Unless
+     *  `holdoutScenarios` reserves an explicit set, ALL scenarios train. */
+    holdout?: 'measured' | 'deferred';
     /** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
     reps?: number;
+    /** DEPTH dial forwarded to the proposer's `propose()` as
+     *  `ctx.maxImprovementShots` — max iterations an agentic candidate generator
+     *  may take per candidate (verify-in-session retries). Unset ⇒ the
+     *  proposer's own default. */
+    maxImprovementShots?: number;
     /** @deprecated Must be 1 when supplied. The loop promotes only a candidate
      *  that replaces its single global incumbent. */
     promoteTopK?: number;
@@ -3801,7 +3834,7 @@ type SelfImproveProgressEvent = {
 } | {
     kind: 'gate.decided';
     decision: string;
-    lift: number;
+    lift?: number;
 } | {
     kind: 'power.estimated';
     n: number;
@@ -3835,6 +3868,18 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
     baselineSurface: MutableSurface;
     /** Budget + loop shape. All fields optional. */
     budget?: SelfImproveBudget;
+    /**
+     * Complete prior measurement of `baselineSurface` over the TRAIN split.
+     * Forwarded to the loop body, which validates its surface hash, scenario
+     * split, seed (42), reps, and coverage, then skips the baseline search
+     * campaign entirely — no baseline dispatch, no resumability lookup. The
+     * train split is `scenarios` minus the holdout split, so premeasure with
+     * exactly that scenario set (explicit `budget.holdoutScenarios`, or
+     * `budget.holdout: 'deferred'` with no reserved set, makes the train split
+     * deterministic). Prior spend stays in the imported campaign aggregates and
+     * is not re-added to this run's cost ledger.
+     */
+    premeasuredBaseline?: PremeasuredOptimizationBaseline<TArtifact, TScenario>;
     /** Custom surface proposer. Default is `gepaProposer` configured from `llm` +
      *  `mutationPrimitives`. */
     proposer?: SurfaceProposer;
@@ -3934,12 +3979,16 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
     findings?: unknown[];
 }
 interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
-    /** Composite mean across all scenarios, baseline run. */
+    /** Composite mean across all scenarios, baseline run. When
+     *  `budget.holdout === 'deferred'` this is measured on the improvement
+     *  (search) split — no holdout campaign ran. */
     baseline: {
         compositeMean: number;
         perScenario: Record<string, number>;
     };
-    /** Composite mean on the held-out set, winner run. */
+    /** Composite mean on the held-out set, winner run. When
+     *  `budget.holdout === 'deferred'` this is the winner's improvement-set
+     *  (search) measurement — no holdout campaign ran. */
     winner: {
         compositeMean: number;
         perScenario: Record<string, number>;
@@ -3953,8 +4002,10 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
         rationale?: string;
     };
     /** `winner.compositeMean - baselineOnHoldout.compositeMean`. Positive
-     *  means the gate observed improvement. */
-    lift: number;
+     *  means the gate observed improvement. Absent iff
+     *  `budget.holdout === 'deferred'` — no held-out measurement ran, so there
+     *  is no lift to report (never a fabricated 0). */
+    lift?: number;
     /** The explicit baseline→winner unified diff. Always present (empty string
      *  when winner == baseline). */
     diff: string;

package/dist/contract/index.js

@@ -36,7 +36,7 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-4EDDIK77.js";
+} from "../chunk-HZJF4IUO.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -125,6 +125,15 @@ function meanComposite(byScenario) {
     perScenario
   };
 }
+function winnerSearchCampaign(result) {
+  for (let i = result.generations.length - 1; i >= 0; i--) {
+    const measured = result.generations[i]?.surfaces.find(
+      (s) => s.surfaceHash === result.winnerSurfaceHash
+    );
+    if (measured) return measured.campaign;
+  }
+  return result.baselineCampaign;
+}
 async function selfImprove(opts) {
   const startedAt = Date.now();
   const requestedRunDir = opts.runDir ?? `mem://selfImprove-${startedAt}`;
@@ -147,18 +156,20 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const populationSize = budget.populationSize ?? 2;
   const maxConcurrency = budget.maxConcurrency ?? 2;
   const holdoutFraction = budget.holdoutFraction ?? 0.25;
+  const holdoutMode = budget.holdout ?? "measured";
+  const holdoutDeferred = holdoutMode === "deferred";
   const expectUsage = opts.expectUsage ?? "assert";
   const explicitHoldout = budget.holdoutScenarios;
   const { train, holdout } = explicitHoldout ? {
     train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
     holdout: explicitHoldout
-  } : splitTrainHoldout(opts.scenarios, holdoutFraction);
+  } : holdoutDeferred ? { train: opts.scenarios, holdout: [] } : splitTrainHoldout(opts.scenarios, holdoutFraction);
   if (train.length === 0) {
     throw new Error(
       "selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios."
     );
   }
-  if (holdout.length === 0) {
+  if (holdout.length === 0 && !holdoutDeferred) {
     throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
   }
   const proposer = opts.proposer ?? gepaProposer({
@@ -182,6 +193,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const result = await runImprovementLoop({
     scenarios: train,
     baselineSurface: opts.baselineSurface,
+    premeasuredBaseline: opts.premeasuredBaseline,
     dispatchWithSurface: opts.agent,
     proposer,
     judges: [opts.judge],
@@ -189,7 +201,9 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     maxGenerations: generations,
     promoteTopK: budget.promoteTopK,
     reps: budget.reps,
+    maxImprovementShots: budget.maxImprovementShots,
     holdoutScenarios: holdout,
+    holdout: holdoutMode,
     gate,
     neutralize: opts.neutralize,
     autoOnPromote: opts.autoOnPromote ?? "none",
@@ -207,8 +221,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     analyzeGeneration: opts.analyzeGeneration,
     findings: opts.findings
   });
-  const baseline = meanComposite(result.baselineOnHoldout.aggregates.byScenario);
-  const winnerStats = meanComposite(result.winnerOnHoldout.aggregates.byScenario);
+  const winnerSearch = holdoutDeferred ? winnerSearchCampaign(result) : void 0;
+  const baseline = meanComposite(
+    (holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout).aggregates.byScenario
+  );
+  const winnerStats = meanComposite((winnerSearch ?? result.winnerOnHoldout).aggregates.byScenario);
   let power;
   const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
     const scores = Object.values(cell.judgeScores);
@@ -241,7 +258,10 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     opts.onProgress({
       kind: "gate.decided",
       decision: result.gateResult.decision,
-      lift: winnerStats.compositeMean - baseline.compositeMean
+      // Deferred holdout has no held-out measurement: in that mode the summary
+      // stats are search-split numbers, and emitting their delta as `lift`
+      // would misreport a train-split delta as a held-out one. Omit instead.
+      ...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean }
     });
   }
   const cost = result.cost;
@@ -249,7 +269,12 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const insight = await analyzeRuns({
     runs: [
       ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
-      ...cellsToRunRecords(result.winnerOnHoldout.cells, "winner", runDir, result.winnerSurface)
+      ...cellsToRunRecords(
+        (winnerSearch ?? result.winnerOnHoldout).cells,
+        "winner",
+        runDir,
+        result.winnerSurface
+      )
     ],
     baselineCandidateId: "baseline",
     candidateCandidateId: "winner"
@@ -278,7 +303,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
       ...result.winnerLabel ? { label: result.winnerLabel } : {},
       ...result.winnerRationale ? { rationale: result.winnerRationale } : {}
     },
-    lift: winnerStats.compositeMean - baseline.compositeMean,
+    ...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean },
     diff: result.promotedDiff,
     provenance,
     gateDecision: result.gateResult.decision,