@tangle-network/agent-eval 0.222.1 → 0.223.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-CSEQRC9l.d.ts → agent-profile-CoD1Dotf.d.ts} +2 -2
- package/dist/{agent-profile-CSEQRC9l.d.ts.map → agent-profile-CoD1Dotf.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +11 -11
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{analyze-runs-CGLRgZGd.d.ts → analyze-runs-C-pP1sK9.d.ts} +2 -2
- package/dist/{analyze-runs-CGLRgZGd.d.ts.map → analyze-runs-C-pP1sK9.d.ts.map} +1 -1
- package/dist/{analyze-runs-CTheASYV.js → analyze-runs-Ck68mbgj.js} +7 -9
- package/dist/analyze-runs-Ck68mbgj.js.map +1 -0
- package/dist/{benchmark-command-Cr7i7YSX.js → benchmark-command-CH86twZj.js} +31 -36
- package/dist/benchmark-command-CH86twZj.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +9 -9
- package/dist/{campaign-ddr1N3yD.js → campaign-DdOgQfp8.js} +14 -19
- package/dist/campaign-DdOgQfp8.js.map +1 -0
- package/dist/{campaign-evidence-DN8rWuBj.js → campaign-evidence-MjCjkb5a.js} +3 -3
- package/dist/{campaign-evidence-DN8rWuBj.js.map → campaign-evidence-MjCjkb5a.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +11 -10
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-1f4ukAIX.d.ts → default-registry-B1DstPYg.d.ts} +7 -7
- package/dist/{default-registry-1f4ukAIX.d.ts.map → default-registry-B1DstPYg.d.ts.map} +1 -1
- package/dist/{define-agent-eval-sUYh9BFA.d.ts → define-agent-eval-CAffiFbN.d.ts} +7 -7
- package/dist/{define-agent-eval-sUYh9BFA.d.ts.map → define-agent-eval-CAffiFbN.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BLoP4Lf9.js → define-agent-eval-Cc15qGik.js} +5 -5
- package/dist/{define-agent-eval-BLoP4Lf9.js.map → define-agent-eval-Cc15qGik.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{engine-BITiErg-.d.ts → engine-CaHaCc3W.d.ts} +2 -2
- package/dist/{engine-BITiErg-.d.ts.map → engine-CaHaCc3W.d.ts.map} +1 -1
- package/dist/{eval-campaign-Cgixrvf7.js → eval-campaign-C_mNDshv.js} +2 -2
- package/dist/{eval-campaign-Cgixrvf7.js.map → eval-campaign-C_mNDshv.js.map} +1 -1
- package/dist/{evaluation-Djq7-Ybj.d.ts → evaluation-BPqUvZ66.d.ts} +3 -3
- package/dist/{evaluation-Djq7-Ybj.d.ts.map → evaluation-BPqUvZ66.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/{exact-types-CndDT_h7.d.ts → exact-types-D98BKpxo.d.ts} +2 -2
- package/dist/{exact-types-CndDT_h7.d.ts.map → exact-types-D98BKpxo.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/experiment/index.js +7 -7
- package/dist/{feedback-trajectory-DsHR9krG.d.ts → feedback-trajectory-SBGncrCP.d.ts} +3 -3
- package/dist/{feedback-trajectory-DsHR9krG.d.ts.map → feedback-trajectory-SBGncrCP.d.ts.map} +1 -1
- package/dist/{index-Bdcpx2s8.d.ts → index-B7v2QHlD.d.ts} +6 -6
- package/dist/{index-Bdcpx2s8.d.ts.map → index-B7v2QHlD.d.ts.map} +1 -1
- package/dist/{index-IzWTz5EP.d.ts → index-CUjeTDPv.d.ts} +6 -6
- package/dist/{index-IzWTz5EP.d.ts.map → index-CUjeTDPv.d.ts.map} +1 -1
- package/dist/index-DAk2quYu.d.ts.map +1 -1
- package/dist/{index-BdKzYYkC.d.ts → index-DUn0GNiZ.d.ts} +2 -2
- package/dist/{index-BdKzYYkC.d.ts.map → index-DUn0GNiZ.d.ts.map} +1 -1
- package/dist/{index-Cu12qxJb.d.ts → index-aDPVYBIM.d.ts} +8 -8
- package/dist/{index-Cu12qxJb.d.ts.map → index-aDPVYBIM.d.ts.map} +1 -1
- package/dist/index.d.ts +17 -17
- package/dist/index.js +13 -12
- package/dist/index.js.map +1 -1
- package/dist/{judge-gate-C2BCOt-M.js → judge-gate-Di_6bifh.js} +4 -4
- package/dist/judge-gate-Di_6bifh.js.map +1 -0
- package/dist/{llm-judge-cgbhTKQp.js → llm-judge-D6JtIqiU.js} +9 -9
- package/dist/{llm-judge-cgbhTKQp.js.map → llm-judge-D6JtIqiU.js.map} +1 -1
- package/dist/{matrix-8X6i5kJ6.d.ts → matrix-CBUzIZs4.d.ts} +2 -2
- package/dist/{matrix-8X6i5kJ6.d.ts.map → matrix-CBUzIZs4.d.ts.map} +1 -1
- package/dist/mean-BPuUNznQ.js +16 -0
- package/dist/mean-BPuUNznQ.js.map +1 -0
- package/dist/meta-eval/index.js +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-D-54hO-k.js → paired-arms-D5H_58e8.js} +2 -2
- package/dist/{paired-arms-D-54hO-k.js.map → paired-arms-D5H_58e8.js.map} +1 -1
- package/dist/{paired-promotion-decision-FGvHA4p_.js → paired-promotion-decision-BiVumgvM.js} +2 -2
- package/dist/{paired-promotion-decision-FGvHA4p_.js.map → paired-promotion-decision-BiVumgvM.js.map} +1 -1
- package/dist/{mean-CdDYG-us.js → paired-tests-C2zUqgUW.js} +2 -7
- package/dist/paired-tests-C2zUqgUW.js.map +1 -0
- package/dist/{pareto-B9udAt2i.js → pareto-7_LEmYGG.js} +4 -3
- package/dist/{pareto-B9udAt2i.js.map → pareto-7_LEmYGG.js.map} +1 -1
- package/dist/{power-and-mde-ClI6RWw4.js → power-and-mde-CHvyDO_H.js} +2 -2
- package/dist/{power-and-mde-ClI6RWw4.js.map → power-and-mde-CHvyDO_H.js.map} +1 -1
- package/dist/{prime-protocol-BpQlTowq.d.ts → prime-protocol-COFkxas6.d.ts} +2 -2
- package/dist/{prime-protocol-BpQlTowq.d.ts.map → prime-protocol-COFkxas6.d.ts.map} +1 -1
- package/dist/{promotion-policy-CBYeD4zi.js → promotion-policy-D7B4_jEZ.js} +4 -4
- package/dist/{promotion-policy-CBYeD4zi.js.map → promotion-policy-D7B4_jEZ.js.map} +1 -1
- package/dist/{registry-D9A7Tybn.d.ts → registry-dhdDoAV8.d.ts} +3 -3
- package/dist/{registry-D9A7Tybn.d.ts.map → registry-dhdDoAV8.d.ts.map} +1 -1
- package/dist/reporting.js +3 -3
- package/dist/{reward-hacking-DZzHZlS1.js → reward-hacking-Bjk3MAjh.js} +7 -9
- package/dist/{reward-hacking-DZzHZlS1.js.map → reward-hacking-Bjk3MAjh.js.map} +1 -1
- package/dist/reward-hacking-C9hPVT0Q.d.ts.map +1 -1
- package/dist/rl.d.ts +3 -3
- package/dist/rl.js +6 -6
- package/dist/rollout/index.d.ts +2 -2
- package/dist/{run-record-CZy4-TnZ.js → run-record-BUm-Jggo.js} +2 -2
- package/dist/{run-record-CZy4-TnZ.js.map → run-record-BUm-Jggo.js.map} +1 -1
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-B77eE7Ac.d.ts → search-kernel-CBdwFZ05.d.ts} +2 -2
- package/dist/{search-kernel-B77eE7Ac.d.ts.map → search-kernel-CBdwFZ05.d.ts.map} +1 -1
- package/dist/{search-summary-Ct7SrmVc.js → search-summary-Bt3uuK53.js} +4 -4
- package/dist/{search-summary-Ct7SrmVc.js.map → search-summary-Bt3uuK53.js.map} +1 -1
- package/dist/{sequential-B1I0tRfL.js → sequential-De7BTAlV.js} +2 -2
- package/dist/{sequential-B1I0tRfL.js.map → sequential-De7BTAlV.js.map} +1 -1
- package/dist/{skillopt-optimization-method-qB1M554K.js → skillopt-optimization-method-BKFbJEyz.js} +9 -12
- package/dist/skillopt-optimization-method-BKFbJEyz.js.map +1 -0
- package/dist/{statistical-heldout-TP6ndL0B.js → statistical-heldout-C9hz_AF9.js} +3 -3
- package/dist/{statistical-heldout-TP6ndL0B.js.map → statistical-heldout-C9hz_AF9.js.map} +1 -1
- package/dist/{statistical-heldout-BYxHe3KO.d.ts → statistical-heldout-Cei036nE.d.ts} +3 -3
- package/dist/{statistical-heldout-BYxHe3KO.d.ts.map → statistical-heldout-Cei036nE.d.ts.map} +1 -1
- package/dist/{store-tool-spans-DMM6wMwB.d.ts → store-tool-spans-B3i3fl0A.d.ts} +4 -4
- package/dist/{store-tool-spans-DMM6wMwB.d.ts.map → store-tool-spans-B3i3fl0A.d.ts.map} +1 -1
- package/dist/storyboard/index.d.ts +17 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +154 -107
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-CPfsrLac.js → summary-report-BLnTyQKH.js} +5 -4
- package/dist/{summary-report-CPfsrLac.js.map → summary-report-BLnTyQKH.js.map} +1 -1
- package/dist/{systemone-UHbKD6LH.d.ts → systemone-Cw2txK5m.d.ts} +5 -5
- package/dist/{systemone-UHbKD6LH.d.ts.map → systemone-Cw2txK5m.d.ts.map} +1 -1
- package/dist/{systemone-review-Cn9Uj0xG.d.ts → systemone-review-BLgxt-4Q.d.ts} +3 -3
- package/dist/{systemone-review-Cn9Uj0xG.d.ts.map → systemone-review-BLgxt-4Q.d.ts.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/{tool-groups-CFWmAGCV.d.ts → tool-groups-C3K-d3gd.d.ts} +3 -3
- package/dist/tool-groups-C3K-d3gd.d.ts.map +1 -0
- package/dist/{trace-questions-BFCQAzoT.d.ts → trace-questions-ej5AEJ5X.d.ts} +3 -3
- package/dist/{trace-questions-BFCQAzoT.d.ts.map → trace-questions-ej5AEJ5X.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/trace-repair/index.js +2 -1
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/{tree-DrIChF4a.js → tree-DnBkWXun.js} +5 -4
- package/dist/{tree-DrIChF4a.js.map → tree-DnBkWXun.js.map} +1 -1
- package/dist/{types-DUR0dqET.d.ts → types-0oMg7zee.d.ts} +2 -2
- package/dist/{types-DUR0dqET.d.ts.map → types-0oMg7zee.d.ts.map} +1 -1
- package/dist/{types-e3HGaI7P.d.ts → types-C54Uk_a1.d.ts} +2 -2
- package/dist/{types-e3HGaI7P.d.ts.map → types-C54Uk_a1.d.ts.map} +1 -1
- package/dist/{types-BEvjDziM.d.ts → types-DzCO-S1m.d.ts} +8 -8
- package/dist/{types-BEvjDziM.d.ts.map → types-DzCO-S1m.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-CTheASYV.js.map +0 -1
- package/dist/benchmark-command-Cr7i7YSX.js.map +0 -1
- package/dist/campaign-ddr1N3yD.js.map +0 -1
- package/dist/judge-gate-C2BCOt-M.js.map +0 -1
- package/dist/mean-CdDYG-us.js.map +0 -1
- package/dist/skillopt-optimization-method-qB1M554K.js.map +0 -1
- package/dist/tool-groups-CFWmAGCV.d.ts.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paired-tests-C2zUqgUW.js","names":[],"sources":["../src/math/student-t.ts","../src/statistics/paired-tests.ts"],"sourcesContent":["import { regularizedIncompleteBeta } from './special-functions'\n\n/**\n * Student-t CDF via the regularized incomplete beta function.\n */\nexport function studentTCdf(t: number, degreesOfFreedom: number): number {\n if (degreesOfFreedom <= 0) return 0.5\n\n const x = degreesOfFreedom / (degreesOfFreedom + t * t)\n const beta = regularizedIncompleteBeta(x, degreesOfFreedom / 2, 0.5)\n return t >= 0 ? 1 - 0.5 * beta : 0.5 * beta\n}\n\n/**\n * Inverse Student-t CDF, solved against {@link studentTCdf}.\n *\n * The CDF is monotone, so bracket expansion followed by bisection is stable\n * across fractional degrees of freedom and does not need a separate\n * approximation with a different error profile.\n */\nexport function studentTQuantile(probability: number, degreesOfFreedom: number): number {\n if (!Number.isFinite(probability) || probability < 0 || probability > 1) {\n throw new RangeError(`studentTQuantile: probability must be in [0,1], got ${probability}`)\n }\n if (!Number.isFinite(degreesOfFreedom) || degreesOfFreedom <= 0) {\n throw new RangeError(\n `studentTQuantile: degreesOfFreedom must be positive and finite, got ${degreesOfFreedom}`,\n )\n }\n if (probability === 0) return Number.NEGATIVE_INFINITY\n if (probability === 1) return Number.POSITIVE_INFINITY\n if (probability === 0.5) return 0\n if (probability < 0.5) return -studentTQuantile(1 - probability, degreesOfFreedom)\n\n let low = 0\n let high = 1\n while (studentTCdf(high, degreesOfFreedom) < probability) high *= 2\n for (let iteration = 0; iteration < 64; iteration++) {\n const middle = (low + high) / 2\n if (studentTCdf(middle, degreesOfFreedom) < probability) low = middle\n else high = middle\n }\n return (low + high) / 2\n}\n","/**\n * Paired significance tests on continuous scores: the paired t-test, the\n * promotion-gate paired bootstrap, the exact sign test, and the package-wide\n * paired-delta decision statistic.\n */\n\nimport { ValidationError } from '../errors'\nimport { studentTCdf } from '../math/student-t'\nimport { assertFiniteSample, binomialHalfUpperTail, makeRng, medianInPlace } from './internal'\n\nexport interface PairedTTestResult {\n /** Null when the statistic is undefined — see {@link pairedTTest}. */\n t: number | null\n df: number\n /** Null exactly when `t` is null. */\n p: number | null\n}\n\n/**\n * Paired t-test — before/after measurements on the SAME items.\n * Pairing removes inter-item variance, giving tighter significance than\n * an unpaired test when comparing prompt v1 vs prompt v2 on identical\n * scenarios.\n *\n * Returns `t = p = null` where the statistic is undefined: fewer than two\n * pairs, or a non-zero constant delta whose observed variance is zero. A\n * constant shift carries no information about the variance it would have to\n * be compared against, so the honest answer is \"undefined\", not `p = 0` —\n * three observations cannot buy absolute certainty. This is the same contract\n * {@link pairedCohensDz} states for the same condition. An all-zero delta is\n * different: it is a measured null, and returns `t = 0, p = 1`.\n */\nexport function pairedTTest(before: number[], after: number[]): PairedTTestResult {\n if (before.length !== after.length) {\n throw new ValidationError(\n `pairedTTest: unequal sample sizes (${before.length} vs ${after.length})`,\n )\n }\n assertFiniteSample('pairedTTest', 'before', before)\n assertFiniteSample('pairedTTest', 'after', after)\n const n = before.length\n if (n < 2) return { t: null, df: 0, p: null }\n\n const diffs = before.map((b, i) => after[i]! - b)\n const mean = diffs.reduce((a, b) => a + b, 0) / n\n const variance = diffs.reduce((acc, d) => acc + (d - mean) ** 2, 0) / (n - 1)\n const se = Math.sqrt(variance / n)\n if (se === 0) {\n return mean === 0 ? { t: 0, df: n - 1, p: 1 } : { t: null, df: n - 1, p: null }\n }\n\n const t = mean / se\n const df = n - 1\n const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n return { t, df, p }\n}\n\n// ── Paired bootstrap (promotion-gate effect size) ────────────────────\n\nexport interface PairedBootstrapResult {\n /** Number of paired observations. */\n n: number\n /** Median of paired deltas (after − before). */\n median: number\n /** Mean of paired deltas. */\n mean: number\n /** Lower bound of the bootstrap CI on the chosen statistic. */\n low: number\n /** Upper bound of the bootstrap CI on the chosen statistic. */\n high: number\n /** Confidence level used (e.g. 0.95). */\n confidence: number\n /** Number of bootstrap resamples used. */\n resamples: number\n /** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */\n gateEligible: boolean\n}\n\n/**\n * Pairs below which a percentile bootstrap interval is descriptive spread only.\n *\n * `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000\n * seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the\n * median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling\n * three points, not an implementation error — scipy's BCa gives 16.0 % on the\n * same n = 3 data — so no change to the estimator moves it. Below this floor\n * the decision belongs to the exact sign test or exact signed-rank test.\n */\nexport const BOOTSTRAP_GATE_MIN_N = 20\n\nexport interface PairedBootstrapOptions {\n /** Confidence level. Default 0.95. */\n confidence?: number\n /** Bootstrap resample count. Default 2000. */\n resamples?: number\n /** Statistic to bootstrap. Default 'median'. */\n statistic?: 'median' | 'mean'\n /** Deterministic seed. If omitted, derived from the deltas so the interval\n * is reproducible regardless. */\n seed?: number\n}\n\n/**\n * Paired bootstrap on (after − before) deltas. Returns a CI on the chosen\n * statistic (median by default); pairs are resampled with replacement. Throws\n * on unequal sample sizes.\n *\n * `low > threshold` carries the stated confidence ONLY at `n ≥\n * {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the\n * check fires under a true null several times more often than nominal, so the\n * interval is descriptive spread and a promotion must not turn on it.\n */\nexport function pairedBootstrap(\n before: number[],\n after: number[],\n opts: PairedBootstrapOptions = {},\n): PairedBootstrapResult {\n if (before.length !== after.length) {\n throw new Error(`pairedBootstrap: unequal sample sizes (${before.length} vs ${after.length})`)\n }\n const confidence = opts.confidence ?? 0.95\n const resamples = opts.resamples ?? 2000\n const statistic = opts.statistic ?? 'median'\n if (confidence <= 0 || confidence >= 1) {\n throw new Error(`pairedBootstrap: confidence must be in (0,1), got ${confidence}`)\n }\n\n const n = before.length\n const deltas = before.map((b, i) => after[i]! - b)\n const gateEligible = n >= BOOTSTRAP_GATE_MIN_N\n if (n === 0) {\n return { n: 0, median: 0, mean: 0, low: 0, high: 0, confidence, resamples, gateEligible }\n }\n if (n === 1) {\n const d = deltas[0]!\n return { n: 1, median: d, mean: d, low: d, high: d, confidence, resamples, gateEligible }\n }\n\n const rng = makeRng(opts.seed, deltas)\n const samples = new Array<number>(resamples)\n for (let b = 0; b < resamples; b++) {\n if (statistic === 'mean') {\n let sum = 0\n for (let k = 0; k < n; k++) {\n sum += deltas[Math.floor(rng() * n)]!\n }\n samples[b] = sum / n\n } else {\n const acc = new Array<number>(n)\n for (let k = 0; k < n; k++) {\n acc[k] = deltas[Math.floor(rng() * n)]!\n }\n samples[b] = medianInPlace(acc)\n }\n }\n samples.sort((a, b) => a - b)\n\n const alpha = 1 - confidence\n const lowIdx = Math.floor((alpha / 2) * resamples)\n const highIdx = Math.min(resamples - 1, Math.ceil((1 - alpha / 2) * resamples) - 1)\n\n return {\n n,\n median: medianInPlace([...deltas]),\n mean: deltas.reduce((s, x) => s + x, 0) / n,\n low: samples[lowIdx]!,\n high: samples[Math.max(highIdx, lowIdx)]!,\n confidence,\n resamples,\n gateEligible,\n }\n}\n\n/** Pre-registered direction for a one-sided paired sign test. */\nexport type SignTestAlternative = 'greater' | 'less'\n\n/** Exact one-sided sign-test result for paired numeric differences. */\nexport interface PairedSignTestResult {\n /** Total supplied differences, including zero ties. */\n n: number\n /** Strictly positive differences. */\n positive: number\n /** Strictly negative differences. */\n negative: number\n /** Zero differences excluded from the binomial test. */\n ties: number\n /** Non-zero differences used by the binomial test. */\n nNonTies: number\n /** Direction of the pre-registered alternative hypothesis. */\n alternative: SignTestAlternative\n /** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */\n pValue: number\n}\n\n/**\n * Exact one-sided sign test over paired differences.\n *\n * Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`\n * tests whether positive signs are more likely than negative signs and returns\n * `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats\n * negative signs as successes instead. With a continuous difference\n * distribution this is the usual directional median test. Exact zero\n * differences are ties and do not enter the binomial denominator. All-tie and\n * empty inputs return p = 1. Every input difference must be finite, and the\n * direction must be chosen explicitly so a caller cannot select it after\n * seeing the signs.\n */\nexport function pairedSignTest(\n differences: readonly number[],\n alternative: SignTestAlternative,\n): PairedSignTestResult {\n if (alternative !== 'greater' && alternative !== 'less') {\n throw new ValidationError(\n `pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`,\n )\n }\n\n let positive = 0\n let negative = 0\n let ties = 0\n for (let i = 0; i < differences.length; i++) {\n const difference = differences[i]!\n if (!Number.isFinite(difference)) {\n throw new ValidationError(\n `pairedSignTest: difference at index ${i} must be finite, got ${difference}`,\n )\n }\n if (difference > 0) positive++\n else if (difference < 0) negative++\n else ties++\n }\n\n const nNonTies = positive + negative\n const successes = alternative === 'greater' ? positive : negative\n return {\n n: differences.length,\n positive,\n negative,\n ties,\n nNonTies,\n alternative,\n pValue: binomialHalfUpperTail(successes, nNonTies),\n }\n}\n\n/** Fraction of paired observations whose delta is an exact tie (|after − before|\n * < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */\nexport function pairedDeltaTieFraction(\n before: ArrayLike<number>,\n after: ArrayLike<number>,\n): number {\n if (before.length !== after.length) {\n throw new Error(\n `pairedDeltaTieFraction: unequal sample sizes (${before.length} vs ${after.length})`,\n )\n }\n const n = before.length\n if (n === 0) return 0\n let ties = 0\n for (let i = 0; i < n; i++) {\n if (Math.abs(after[i]! - before[i]!) < 1e-9) ties++\n }\n return ties / n\n}\n\n/**\n * The paired-delta statistic a DECISION is computed on, package-wide.\n *\n * The mean paired delta is the estimator that answers the question a promotion\n * gate asks — \"by how much did the candidate move the score\" — in the caller's\n * own units, and it equals the aggregate lift everyone quotes. The MEDIAN\n * answers a different question and loses the answer to this one in every regime\n * eval data actually lands in:\n * - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in\n * {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI\n * are pinned at exactly 0 however large the shift. (Decide these on\n * {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)\n * - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by\n * construction, and `ci.low > threshold` then answers \"no\" forever at a\n * non-negative threshold and \"yes\" forever at a negative one.\n * - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on\n * integer 0-100, and block scores like {⅔, 1} from averaging pass/fail\n * leaves, put the median on a coarse lattice whose bootstrap percentiles\n * land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real\n * +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —\n * lower bound exactly 0, so a gate at threshold 0 refuses a real lift.\n * That last case is why there is no tie-fraction threshold here: any cutoff on\n * ties leaves the lattice case open on the other side of it.\n *\n * `heldoutSignificance` has defaulted to the mean since #316 for the same\n * reason. The median remains available per call site for callers who\n * specifically want outlier robustness and accept the blindness.\n */\nexport const DECISION_PAIRED_DELTA_STATISTIC: 'mean' = 'mean'\n"],"mappings":";;;;;;AAKA,SAAgB,YAAY,GAAW,kBAAkC;CACvE,IAAI,oBAAoB,GAAG,OAAO;CAElC,MAAM,IAAI,oBAAoB,mBAAmB,IAAI;CACrD,MAAM,OAAO,0BAA0B,GAAG,mBAAmB,GAAG,EAAG;CACnE,OAAO,KAAK,IAAI,IAAI,KAAM,OAAO,KAAM;AACzC;;;;;;;;AASA,SAAgB,iBAAiB,aAAqB,kBAAkC;CACtF,IAAI,CAAC,OAAO,SAAS,WAAW,KAAK,cAAc,KAAK,cAAc,GACpE,MAAM,IAAI,WAAW,uDAAuD,aAAa;CAE3F,IAAI,CAAC,OAAO,SAAS,gBAAgB,KAAK,oBAAoB,GAC5D,MAAM,IAAI,WACR,uEAAuE,kBACzE;CAEF,IAAI,gBAAgB,GAAG,OAAO,OAAO;CACrC,IAAI,gBAAgB,GAAG,OAAO,OAAO;CACrC,IAAI,gBAAgB,IAAK,OAAO;CAChC,IAAI,cAAc,IAAK,OAAO,CAAC,iBAAiB,IAAI,aAAa,gBAAgB;CAEjF,IAAI,MAAM;CACV,IAAI,OAAO;CACX,OAAO,YAAY,MAAM,gBAAgB,IAAI,aAAa,QAAQ;CAClE,KAAK,IAAI,YAAY,GAAG,YAAY,IAAI,aAAa;EACnD,MAAM,UAAU,MAAM,QAAQ;EAC9B,IAAI,YAAY,QAAQ,gBAAgB,IAAI,aAAa,MAAM;OAC1D,OAAO;CACd;CACA,QAAQ,MAAM,QAAQ;AACxB;;;;;;;;;;;;;;;;;;;;;ACXA,SAAgB,YAAY,QAAkB,OAAoC;CAChF,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,gBACR,sCAAsC,OAAO,OAAO,MAAM,MAAM,OAAO,EACzE;CAEF,mBAAmB,eAAe,UAAU,MAAM;CAClD,mBAAmB,eAAe,SAAS,KAAK;CAChD,MAAM,IAAI,OAAO;CACjB,IAAI,IAAI,GAAG,OAAO;EAAE,GAAG;EAAM,IAAI;EAAG,GAAG;CAAK;CAE5C,MAAM,QAAQ,OAAO,KAAK,GAAG,MAAM,MAAM,KAAM,CAAC;CAChD,MAAM,OAAO,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CAChD,MAAM,WAAW,MAAM,QAAQ,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC,KAAK,IAAI;CAC3E,MAAM,KAAK,KAAK,KAAK,WAAW,CAAC;CACjC,IAAI,OAAO,GACT,OAAO,SAAS,IAAI;EAAE,GAAG;EAAG,IAAI,IAAI;EAAG,GAAG;CAAE,IAAI;EAAE,GAAG;EAAM,IAAI,IAAI;EAAG,GAAG;CAAK;CAGhF,MAAM,IAAI,OAAO;CACjB,MAAM,KAAK,IAAI;CAEf,OAAO;EAAE;EAAG;EAAI,GADN,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;CAC5B;AACpB;;;;;;;;;;;AAiCA,MAAa,uBAAuB;;;;;;;;;;;AAwBpC,SAAgB,gBACd,QACA,OACA,OAA+B,CAAC,GACT;CACvB,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,MAAM,0CAA0C,OAAO,OAAO,MAAM,MAAM,OAAO,EAAE;CAE/F,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,YAAY,KAAK,aAAa;CACpC,IAAI,cAAc,KAAK,cAAc,GACnC,MAAM,IAAI,MAAM,qDAAqD,YAAY;CAGnF,MAAM,IAAI,OAAO;CACjB,MAAM,SAAS,OAAO,KAAK,GAAG,MAAM,MAAM,KAAM,CAAC;CACjD,MAAM,eAAe,KAAA;CACrB,IAAI,MAAM,GACR,OAAO;EAAE,GAAG;EAAG,QAAQ;EAAG,MAAM;EAAG,KAAK;EAAG,MAAM;EAAG;EAAY;EAAW;CAAa;CAE1F,IAAI,MAAM,GAAG;EACX,MAAM,IAAI,OAAO;EACjB,OAAO;GAAE,GAAG;GAAG,QAAQ;GAAG,MAAM;GAAG,KAAK;GAAG,MAAM;GAAG;GAAY;GAAW;EAAa;CAC1F;CAEA,MAAM,MAAM,QAAQ,KAAK,MAAM,MAAM;CACrC,MAAM,UAAU,IAAI,MAAc,SAAS;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAC7B,IAAI,cAAc,QAAQ;EACxB,IAAI,MAAM;EACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,OAAO,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC;EAEpC,QAAQ,KAAK,MAAM;CACrB,OAAO;EACL,MAAM,MAAM,IAAI,MAAc,CAAC;EAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,IAAI,KAAK,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC;EAEtC,QAAQ,KAAK,cAAc,GAAG;CAChC;CAEF,QAAQ,MAAM,GAAG,MAAM,IAAI,CAAC;CAE5B,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,KAAK,MAAO,QAAQ,IAAK,SAAS;CACjD,MAAM,UAAU,KAAK,IAAI,YAAY,GAAG,KAAK,MAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC;CAElF,OAAO;EACL;EACA,QAAQ,cAAc,CAAC,GAAG,MAAM,CAAC;EACjC,MAAM,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;EAC1C,KAAK,QAAQ;EACb,MAAM,QAAQ,KAAK,IAAI,SAAS,MAAM;EACtC;EACA;EACA;CACF;AACF;;;;;;;;;;;;;;AAoCA,SAAgB,eACd,aACA,aACsB;CACtB,IAAI,gBAAgB,aAAa,gBAAgB,QAC/C,MAAM,IAAI,gBACR,gEAAgE,aAClE;CAGF,IAAI,WAAW;CACf,IAAI,WAAW;CACf,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;EAC3C,MAAM,aAAa,YAAY;EAC/B,IAAI,CAAC,OAAO,SAAS,UAAU,GAC7B,MAAM,IAAI,gBACR,uCAAuC,EAAE,uBAAuB,YAClE;EAEF,IAAI,aAAa,GAAG;OACf,IAAI,aAAa,GAAG;OACpB;CACP;CAEA,MAAM,WAAW,WAAW;CAC5B,MAAM,YAAY,gBAAgB,YAAY,WAAW;CACzD,OAAO;EACL,GAAG,YAAY;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ,sBAAsB,WAAW,QAAQ;CACnD;AACF;;;AAIA,SAAgB,uBACd,QACA,OACQ;CACR,IAAI,OAAO,WAAW,MAAM,QAC1B,MAAM,IAAI,MACR,iDAAiD,OAAO,OAAO,MAAM,MAAM,OAAO,EACpF;CAEF,MAAM,IAAI,OAAO;CACjB,IAAI,MAAM,GAAG,OAAO;CACpB,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,IAAI,KAAK,IAAI,MAAM,KAAM,OAAO,EAAG,IAAI,MAAM;CAE/C,OAAO,OAAO;AAChB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAa,kCAA0C"}
|
|
@@ -2,8 +2,9 @@ import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from
|
|
|
2
2
|
import { t as mulberry32 } from "./random-DrzcgC7F.js";
|
|
3
3
|
import { t as confidenceInterval } from "./descriptive-cPwITVfy.js";
|
|
4
4
|
import { s as wilson } from "./paired-binary-BR0nwHOA.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { n as DECISION_PAIRED_DELTA_STATISTIC } from "./paired-tests-C2zUqgUW.js";
|
|
6
|
+
import { t as mean } from "./mean-BPuUNznQ.js";
|
|
7
|
+
import { i as pairedDeltaTest, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-BiVumgvM.js";
|
|
7
8
|
import { a as SearchLedgerError } from "./search-ledger-file-DTBD6X-y.js";
|
|
8
9
|
import { s as redactText } from "./redact-D6APJS1d.js";
|
|
9
10
|
import { l as searchUnitScores, p as USD_TOLERANCE, t as FileSearchLedger, u as searchCellId } from "./search-ledger-BsVi-Ngv.js";
|
|
@@ -2608,4 +2609,4 @@ function paretoFrontierWithCrowding(candidates, objectives) {
|
|
|
2608
2609
|
}
|
|
2609
2610
|
export { normalizeSignedZero as A, SEARCH_ESTIMATOR as C, searchEstimateMethod as D, searchCellSetDigest as E, searchPosterior as O, requiredPairsForPairedPromotion as S, estimateNodeFromCells as T, planSearchClaim as _, searchDivergence as a, verifySearchClaim as b, MIN_OUTCOMES_FOR_WEIGHT as c, rankingSplit as d, summarizeSamples as f, decideSearchClaim as g, SEARCH_CLAIM_RULE as h, runSearch as i, seedFromDigest as k, operatorYield as l, searchExpansionIndex as m, paretoFrontier as n, searchPolicyView as o, SEARCH_KERNEL_SOURCE as p, paretoFrontierWithCrowding as r, EXPANSION_OPERATORS as s, dominates as t, INSUFFICIENT_FROM as u, searchClaimDecision as v, estimateNode as w, pairedPromotionPower as x, searchClaimReserveUsd as y };
|
|
2610
2611
|
|
|
2611
|
-
//# sourceMappingURL=pareto-
|
|
2612
|
+
//# sourceMappingURL=pareto-7_LEmYGG.js.map
|