@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/chunk-MAZ26DC7.js

@@ -1,99 +0,0 @@
-// src/sequential.ts
-function pairedEvalueSequence(deltas, opts = {}) {
-  const c = opts.bound ?? 1;
-  const alpha = opts.alpha ?? 0.05;
-  const initialShrink = opts.initialBetShrinkage ?? 0.5;
-  const rope = opts.rope ?? null;
-  if (c <= 0) throw new Error("pairedEvalueSequence: bound must be > 0");
-  if (alpha <= 0 || alpha >= 1) throw new Error("pairedEvalueSequence: alpha must be in (0,1)");
-  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {
-    throw new Error("pairedEvalueSequence: rope must satisfy low \u2264 high");
-  }
-  const steps = [];
-  let clipped = false;
-  let evalue = 1;
-  let decisionFiredAt = null;
-  let sum = 0;
-  let sumSq = 0;
-  let count = 0;
-  for (let i = 0; i < deltas.length; i++) {
-    let d = deltas[i];
-    if (d < -c || d > c) {
-      d = Math.max(-c, Math.min(c, d));
-      clipped = true;
-    }
-    const muHat = count === 0 ? 0 : sum / count;
-    const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat);
-    const t = i + 1;
-    const shrink = initialShrink * Math.min(1, count / 32);
-    let lambda = muHat / (varHat + c * c) * shrink;
-    const lambdaMax = 0.99 / c;
-    if (lambda > lambdaMax) lambda = lambdaMax;
-    if (lambda < -lambdaMax) lambda = -lambdaMax;
-    evalue = evalue * (1 + lambda * d);
-    if (!Number.isFinite(evalue) || evalue < 0) evalue = 0;
-    sum += d;
-    sumSq += d * d;
-    count += 1;
-    const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300));
-    const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha);
-    let decision = "continue";
-    if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = "equivalent";
-    else if (evalue >= 2 / alpha && muHat > 0) decision = "promote_now";
-    else if (evalue >= 2 / alpha && muHat < 0) decision = "reject_now";
-    else if (rope && cs.high < rope.low) decision = "reject_now";
-    if (decision !== "continue" && decisionFiredAt === null) decisionFiredAt = t;
-    steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision });
-  }
-  const finalDecision = steps.length === 0 ? "continue" : steps[steps.length - 1].decision;
-  return { steps, finalDecision, decisionFiredAt, clipped };
-}
-function evaluateInterimReleaseConfidence(input) {
-  const candidates = input.deltaSeries.map((s) => {
-    const seq = pairedEvalueSequence(s.deltas, {
-      alpha: input.alpha,
-      bound: input.bound,
-      rope: input.rope
-    });
-    const last = seq.steps[seq.steps.length - 1];
-    return {
-      candidateId: s.candidateId,
-      decision: seq.finalDecision,
-      decisionFiredAt: seq.decisionFiredAt,
-      finalEvalue: last?.evalue ?? 1,
-      finalPValue: last?.pValue ?? 1,
-      pairs: seq.steps.length,
-      csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,
-      csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY
-    };
-  });
-  const promote = candidates.find((c) => c.decision === "promote_now");
-  if (promote)
-    return {
-      candidates,
-      recommendation: { decision: "promote_now", candidateId: promote.candidateId }
-    };
-  const live = candidates.find((c) => c.decision === "continue");
-  if (live) return { candidates, recommendation: { decision: "continue", candidateId: null } };
-  const equiv = candidates.find((c) => c.decision === "equivalent");
-  if (equiv)
-    return {
-      candidates,
-      recommendation: { decision: "equivalent", candidateId: equiv.candidateId }
-    };
-  return { candidates, recommendation: { decision: "reject_now", candidateId: null } };
-}
-function empiricalBernsteinCs(sum, sumSq, n, bound, alpha) {
-  if (n === 0) return { low: -bound, high: bound };
-  const mean = sum / n;
-  const variance = Math.max(0, sumSq / n - mean * mean);
-  const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1);
-  const radius = Math.sqrt(2 * variance * psi / n) + 3 * bound * psi / n;
-  return { low: mean - radius, high: mean + radius };
-}
-
-export {
-  pairedEvalueSequence,
-  evaluateInterimReleaseConfidence
-};
-//# sourceMappingURL=chunk-MAZ26DC7.js.map

package/dist/chunk-MAZ26DC7.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/sequential.ts"],"sourcesContent":["/**\n * Always-valid sequential evaluation.\n *\n * `researchReport` assumes a single pre-specified analysis. Real\n * consumers run campaigns weekly / nightly / per-PR; each new run silently\n * inflates the false-discovery rate, because the BH-FDR guarantee is for\n * the *first* look, not the 47th. Without time-uniform inference,\n * launch-decision teams either (a) don't peek, which forfeits the cost\n * advantage of stop-when-decisive, or (b) peek and pretend they didn't,\n * which forfeits scientific validity.\n *\n * This module ships **e-value-based confidence sequences** for paired\n * bounded outcomes. The methodology is the predictable plug-in betting\n * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*\n * stopping time. Concretely:\n *\n *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,\n *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable\n *   plug-in), and the running e-value is\n *\n *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)\n *\n *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by\n *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null\n *   at any time without inflating the type-I error.\n *\n * Combined with `runEvalCampaign`, every consumer running rolling\n * campaigns gains the ability to ship the moment evidence is decisive,\n * stop-early on dead-on-arrival variants, and accumulate evidence across\n * partial runs without spending the FDR budget. No new sweep is wasted.\n *\n * References:\n *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).\n *     Time-uniform, nonparametric, nonasymptotic confidence sequences.\n *     Annals of Statistics, 49(2), 1055–1080.\n *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded\n *     random variables by betting. JRSS B, 86(1), 1–27.\n */\n\nexport type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent'\n\nexport interface PairedEvalueOptions {\n  /**\n   * Bound on |delta|. Default 1 (matching most score scales). Must satisfy\n   * c > 0; deltas outside [-c, c] are clipped with a warning attached to\n   * the return value.\n   */\n  bound?: number\n  /** Target Type-I error. Default 0.05. */\n  alpha?: number\n  /**\n   * Region of Practical Equivalence on the *mean* paired delta. When\n   * supplied, the verdict can return `'equivalent'` once the running\n   * confidence sequence on the mean is fully contained in [low, high].\n   */\n  rope?: { low: number; high: number }\n  /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */\n  initialBetShrinkage?: number\n}\n\nexport interface PairedEvalueStep {\n  /** 1-indexed observation count. */\n  t: number\n  delta: number\n  /** Running e-value E_t = ∏ (1 + λ_i · D_i). */\n  evalue: number\n  /** Time-uniform p-value at stopping time t. */\n  pValue: number\n  /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */\n  csLow: number\n  csHigh: number\n  /** Verdict at this stopping time. */\n  decision: SequentialDecision\n}\n\nexport interface PairedEvalueSequence {\n  steps: PairedEvalueStep[]\n  /** The decision at the final step. */\n  finalDecision: SequentialDecision\n  /** Index (1-based) at which a non-`continue` decision first fired, or null. */\n  decisionFiredAt: number | null\n  /** True if any deltas were clipped to [-bound, bound]. */\n  clipped: boolean\n}\n\n/**\n * Run the paired e-value sequence over an in-order delta stream.\n *\n * Use for *streaming* / interim analyses: pass the deltas you have so\n * far, get the verdict at every prefix length. The decision is\n * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`\n * fires, the verdict at later steps remains decisive (the e-value is a\n * non-negative martingale; once it crosses the threshold, it's crossed).\n */\nexport function pairedEvalueSequence(\n  deltas: number[],\n  opts: PairedEvalueOptions = {},\n): PairedEvalueSequence {\n  const c = opts.bound ?? 1\n  const alpha = opts.alpha ?? 0.05\n  const initialShrink = opts.initialBetShrinkage ?? 0.5\n  const rope = opts.rope ?? null\n  if (c <= 0) throw new Error('pairedEvalueSequence: bound must be > 0')\n  if (alpha <= 0 || alpha >= 1) throw new Error('pairedEvalueSequence: alpha must be in (0,1)')\n  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {\n    throw new Error('pairedEvalueSequence: rope must satisfy low ≤ high')\n  }\n\n  const steps: PairedEvalueStep[] = []\n  let clipped = false\n  let evalue = 1\n  let decisionFiredAt: number | null = null\n\n  // Running statistics (using only D_{1..i-1} for the bet → predictable plug-in).\n  let sum = 0\n  let sumSq = 0\n  let count = 0\n\n  for (let i = 0; i < deltas.length; i++) {\n    let d = deltas[i]!\n    if (d < -c || d > c) {\n      d = Math.max(-c, Math.min(c, d))\n      clipped = true\n    }\n\n    // Predictable plug-in bet (positive λ tests for E[D] > 0; we run a two-sided\n    // test by tracking the symmetric e-value via |bet|).\n    // λ_i ∝ mean / (variance + bound^2). Shrink early to avoid overbetting.\n    const muHat = count === 0 ? 0 : sum / count\n    const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat)\n    const t = i + 1\n    const shrink = initialShrink * Math.min(1, count / 32) // anneal toward 1\n    let lambda = (muHat / (varHat + c * c)) * shrink\n    // Clip to ensure 1 + λ·D > 0 for all |D| ≤ c (so the e-value stays non-negative).\n    const lambdaMax = 0.99 / c\n    if (lambda > lambdaMax) lambda = lambdaMax\n    if (lambda < -lambdaMax) lambda = -lambdaMax\n\n    evalue = evalue * (1 + lambda * d)\n    if (!Number.isFinite(evalue) || evalue < 0) evalue = 0\n\n    sum += d\n    sumSq += d * d\n    count += 1\n\n    const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300))\n\n    // Empirical Bernstein confidence sequence on the mean. Howard et al.\n    // (2021), Theorem 4.4 with σ̂² the running sample variance and a\n    // calibration constant tuned for two-sided coverage at level 1 - α.\n    const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha)\n\n    let decision: SequentialDecision = 'continue'\n    if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = 'equivalent'\n    else if (evalue >= 2 / alpha && muHat > 0) decision = 'promote_now'\n    else if (evalue >= 2 / alpha && muHat < 0) decision = 'reject_now'\n    else if (rope && cs.high < rope.low) decision = 'reject_now'\n\n    if (decision !== 'continue' && decisionFiredAt === null) decisionFiredAt = t\n\n    steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision })\n  }\n\n  const finalDecision = steps.length === 0 ? 'continue' : steps[steps.length - 1]!.decision\n  return { steps, finalDecision, decisionFiredAt, clipped }\n}\n\nexport interface InterimReleaseConfidenceInput {\n  /**\n   * One delta series per candidate (paired deltas vs comparator). Order\n   * within a series is the order the campaigns were run.\n   */\n  deltaSeries: Array<{ candidateId: string; deltas: number[] }>\n  alpha?: number\n  bound?: number\n  rope?: { low: number; high: number }\n}\n\nexport interface InterimReleaseConfidence {\n  candidates: Array<{\n    candidateId: string\n    decision: SequentialDecision\n    decisionFiredAt: number | null\n    finalEvalue: number\n    finalPValue: number\n    pairs: number\n    csLow: number\n    csHigh: number\n  }>\n  /**\n   * Campaign-level recommendation: pick the strongest 'promote_now', else\n   * 'continue' if any candidate is still live, else 'reject_now' if every\n   * candidate is dead, else 'equivalent'.\n   */\n  recommendation: { decision: SequentialDecision; candidateId: string | null }\n}\n\n/**\n * Run interim sequential analyses across many candidates at once,\n * preserving the time-uniform α guarantee for each candidate's series and\n * synthesising a campaign-level recommendation. Designed to be called on\n * every campaign tick — the recommendation is anytime-valid.\n */\nexport function evaluateInterimReleaseConfidence(\n  input: InterimReleaseConfidenceInput,\n): InterimReleaseConfidence {\n  const candidates = input.deltaSeries.map((s) => {\n    const seq = pairedEvalueSequence(s.deltas, {\n      alpha: input.alpha,\n      bound: input.bound,\n      rope: input.rope,\n    })\n    const last = seq.steps[seq.steps.length - 1]\n    return {\n      candidateId: s.candidateId,\n      decision: seq.finalDecision,\n      decisionFiredAt: seq.decisionFiredAt,\n      finalEvalue: last?.evalue ?? 1,\n      finalPValue: last?.pValue ?? 1,\n      pairs: seq.steps.length,\n      csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,\n      csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY,\n    }\n  })\n\n  const promote = candidates.find((c) => c.decision === 'promote_now')\n  if (promote)\n    return {\n      candidates,\n      recommendation: { decision: 'promote_now', candidateId: promote.candidateId },\n    }\n  const live = candidates.find((c) => c.decision === 'continue')\n  if (live) return { candidates, recommendation: { decision: 'continue', candidateId: null } }\n  const equiv = candidates.find((c) => c.decision === 'equivalent')\n  if (equiv)\n    return {\n      candidates,\n      recommendation: { decision: 'equivalent', candidateId: equiv.candidateId },\n    }\n  return { candidates, recommendation: { decision: 'reject_now', candidateId: null } }\n}\n\n// ── Internals ────────────────────────────────────────────────────────────\n\n/**\n * Empirical Bernstein confidence sequence on the mean of bounded variables.\n * Adapted from Howard et al. (2021) §4.4. Provides a time-uniform CI on\n * the running mean; valid at every stopping time.\n */\nfunction empiricalBernsteinCs(\n  sum: number,\n  sumSq: number,\n  n: number,\n  bound: number,\n  alpha: number,\n): { low: number; high: number } {\n  if (n === 0) return { low: -bound, high: bound }\n  const mean = sum / n\n  const variance = Math.max(0, sumSq / n - mean * mean)\n  // Iterated-log calibration constant. The 1.7 exponent matches the\n  // recommended choice in Howard et al. for two-sided coverage at level\n  // 1 - α with mild log-corrections; tightening further requires a\n  // tuned mixture and is out of scope.\n  const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1)\n  const radius = Math.sqrt((2 * variance * psi) / n) + (3 * bound * psi) / n\n  return { low: mean - radius, high: mean + radius }\n}\n"],"mappings":";AA8FO,SAAS,qBACd,QACA,OAA4B,CAAC,GACP;AACtB,QAAM,IAAI,KAAK,SAAS;AACxB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,gBAAgB,KAAK,uBAAuB;AAClD,QAAM,OAAO,KAAK,QAAQ;AAC1B,MAAI,KAAK,EAAG,OAAM,IAAI,MAAM,yCAAyC;AACrE,MAAI,SAAS,KAAK,SAAS,EAAG,OAAM,IAAI,MAAM,8CAA8C;AAC5F,MAAI,QAAQ,EAAE,OAAO,SAAS,KAAK,GAAG,KAAK,OAAO,SAAS,KAAK,IAAI,KAAK,KAAK,OAAO,KAAK,OAAO;AAC/F,UAAM,IAAI,MAAM,yDAAoD;AAAA,EACtE;AAEA,QAAM,QAA4B,CAAC;AACnC,MAAI,UAAU;AACd,MAAI,SAAS;AACb,MAAI,kBAAiC;AAGrC,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,QAAQ;AAEZ,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,QAAI,IAAI,OAAO,CAAC;AAChB,QAAI,IAAI,CAAC,KAAK,IAAI,GAAG;AACnB,UAAI,KAAK,IAAI,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AAC/B,gBAAU;AAAA,IACZ;AAKA,UAAM,QAAQ,UAAU,IAAI,IAAI,MAAM;AACtC,UAAM,SAAS,UAAU,IAAI,IAAI,IAAI,KAAK,IAAI,OAAO,QAAQ,QAAQ,QAAQ,KAAK;AAClF,UAAM,IAAI,IAAI;AACd,UAAM,SAAS,gBAAgB,KAAK,IAAI,GAAG,QAAQ,EAAE;AACrD,QAAI,SAAU,SAAS,SAAS,IAAI,KAAM;AAE1C,UAAM,YAAY,OAAO;AACzB,QAAI,SAAS,UAAW,UAAS;AACjC,QAAI,SAAS,CAAC,UAAW,UAAS,CAAC;AAEnC,aAAS,UAAU,IAAI,SAAS;AAChC,QAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,UAAS;AAErD,WAAO;AACP,aAAS,IAAI;AACb,aAAS;AAET,UAAM,SAAS,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,MAAM,CAAC;AAKvD,UAAM,KAAK,qBAAqB,KAAK,OAAO,OAAO,GAAG,KAAK;AAE3D,QAAI,WAA+B;AACnC,QAAI,QAAQ,GAAG,OAAO,KAAK,OAAO,GAAG,QAAQ,KAAK,KAAM,YAAW;AAAA,aAC1D,UAAU,IAAI,SAAS,QAAQ,EAAG,YAAW;AAAA,aAC7C,UAAU,IAAI,SAAS,QAAQ,EAAG,YAAW;AAAA,aAC7C,QAAQ,GAAG,OAAO,KAAK,IAAK,YAAW;AAEhD,QAAI,aAAa,cAAc,oBAAoB,KAAM,mBAAkB;AAE3E,UAAM,KAAK,EAAE,GAAG,OAAO,GAAG,QAAQ,QAAQ,OAAO,GAAG,KAAK,QAAQ,GAAG,MAAM,SAAS,CAAC;AAAA,EACtF;AAEA,QAAM,gBAAgB,MAAM,WAAW,IAAI,aAAa,MAAM,MAAM,SAAS,CAAC,EAAG;AACjF,SAAO,EAAE,OAAO,eAAe,iBAAiB,QAAQ;AAC1D;AAsCO,SAAS,iCACd,OAC0B;AAC1B,QAAM,aAAa,MAAM,YAAY,IAAI,CAAC,MAAM;AAC9C,UAAM,MAAM,qBAAqB,EAAE,QAAQ;AAAA,MACzC,OAAO,MAAM;AAAA,MACb,OAAO,MAAM;AAAA,MACb,MAAM,MAAM;AAAA,IACd,CAAC;AACD,UAAM,OAAO,IAAI,MAAM,IAAI,MAAM,SAAS,CAAC;AAC3C,WAAO;AAAA,MACL,aAAa,EAAE;AAAA,MACf,UAAU,IAAI;AAAA,MACd,iBAAiB,IAAI;AAAA,MACrB,aAAa,MAAM,UAAU;AAAA,MAC7B,aAAa,MAAM,UAAU;AAAA,MAC7B,OAAO,IAAI,MAAM;AAAA,MACjB,OAAO,MAAM,SAAS,OAAO;AAAA,MAC7B,QAAQ,MAAM,UAAU,OAAO;AAAA,IACjC;AAAA,EACF,CAAC;AAED,QAAM,UAAU,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,aAAa;AACnE,MAAI;AACF,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,EAAE,UAAU,eAAe,aAAa,QAAQ,YAAY;AAAA,IAC9E;AACF,QAAM,OAAO,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU;AAC7D,MAAI,KAAM,QAAO,EAAE,YAAY,gBAAgB,EAAE,UAAU,YAAY,aAAa,KAAK,EAAE;AAC3F,QAAM,QAAQ,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,YAAY;AAChE,MAAI;AACF,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,EAAE,UAAU,cAAc,aAAa,MAAM,YAAY;AAAA,IAC3E;AACF,SAAO,EAAE,YAAY,gBAAgB,EAAE,UAAU,cAAc,aAAa,KAAK,EAAE;AACrF;AASA,SAAS,qBACP,KACA,OACA,GACA,OACA,OAC+B;AAC/B,MAAI,MAAM,EAAG,QAAO,EAAE,KAAK,CAAC,OAAO,MAAM,MAAM;AAC/C,QAAM,OAAO,MAAM;AACnB,QAAM,WAAW,KAAK,IAAI,GAAG,QAAQ,IAAI,OAAO,IAAI;AAKpD,QAAM,MAAM,KAAK,IAAI,IAAI,KAAK,IAAI,MAAM,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC;AAClF,QAAM,SAAS,KAAK,KAAM,IAAI,WAAW,MAAO,CAAC,IAAK,IAAI,QAAQ,MAAO;AACzE,SAAO,EAAE,KAAK,OAAO,QAAQ,MAAM,OAAO,OAAO;AACnD;","names":[]}