bayes-ab 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -23,7 +23,7 @@ Bayes action now Ship B (expected regret 0.0050 pp, about 0.50 missed conve
23
23
  P(B beats A) 95.91%
24
24
  P(relative lift > +10%) 46.42%
25
25
  P(relative lift < -10%) 0.02%
26
- P(B is harmful at all) 4.06%
26
+ P(B is harmful at all) 4.09%
27
27
 
28
28
  Relative lift +9.54% 95% CI -1.17% to +21.30%
29
29
  Absolute difference +0.53 pp 95% CI -0.07 pp to +1.11 pp
@@ -98,6 +98,14 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 10000 --rat
98
98
  # Continue test · P(B beats A) 97.13% · P(lift > +10%) 56.76% · lift +10.92% (CI -0.30% to +23.39%) · loss if ship B 0.0034 pp B = 610/10000 from --rate-b 6.10%
99
99
  ```
100
100
 
101
+ Choose a prior. Every analysis defaults to a flat Beta(1,1) on each conversion rate; `--prior-alpha` and `--prior-beta` replace it, the same prior for both variants. Beta(α, β) reads as α conversions in α + β visitors of evidence you already had, so a prior worth 1,000 visitors centred on 5.5% is Beta(55, 945). Fractional values work, e.g. the Jeffreys prior Beta(0.5, 0.5):
102
+
103
+ ```sh
104
+ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --conversions-b 714 --prior-alpha 55 --prior-beta 945
105
+ ```
106
+
107
+ A shared prior pulls both arms toward the same rate, so a strong one shrinks the estimated lift and makes decisions more conservative.
108
+
101
109
  ### project
102
110
 
103
111
  Takes the current posterior, draws a plausible pair of true conversion rates, simulates each future day of traffic, and re-runs the decision rule after every day. Repeats that thousands of times.
@@ -173,16 +181,16 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --con
173
181
  ```json
174
182
  {
175
183
  "tool": "bayes-ab",
176
- "version": "0.1.0",
184
+ "version": "0.3.0",
177
185
  "command": "analyze",
178
- "inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 112396 },
186
+ "inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 116218 },
179
187
  "decision": { "status": "continue", "label": "Continue test", "evidence": "low", "maturity": "building", "maturityProgress": 0.462241, "bayesActionNow": "B", "expectedRegretNow": 0.00005 },
180
- "probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040583, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
181
- "effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253 },
188
+ "probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040886, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
189
+ "effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253, "relativeLiftMedian": 0.095267, "relativeLiftIQR": [0.056954, 0.134715] },
182
190
  "expectedLoss": { "shipA": 0.00528, "shipB": 0.00005 },
183
191
  "variants": { "A": { "...": "..." }, "B": { "...": "..." } },
184
192
  "sampleSize": { "requiredPerVariant": 25636, "additionalPerVariant": 13786, "detectableEffectNow": 0.147083 },
185
- "waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.00006, "valueOfWaiting": 0 }, "..." ],
193
+ "waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.000048, "valueOfWaiting": 0.000002 }, "..." ],
186
194
  "frequentist": { "pValueTwoSided": 0.081709 },
187
195
  "shareUrl": "https://www.joebrundage.com/tools/bayesian-ab-test?visitorsA=12000&conversionsA=660&visitorsB=11850&conversionsB=714&thresholdPercent=95&meaningfulLiftPercent=10"
188
196
  }
@@ -227,12 +235,30 @@ import { analyze, project, plan } from "bayes-ab"
227
235
 
228
236
  const result = analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850, conversionsB: 714 })
229
237
  result.probability.bBeatsA // 0.9591138673472563
238
+
239
+ analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850, conversionsB: 714, prior: [55, 945] })
230
240
  ```
231
241
 
242
+ `analyze` and `project` take an optional `prior: [alpha, beta]`, default `[1, 1]`. Non-positive values throw a `RangeError`; `validatePrior` and `isDefaultPrior` are exported for forms that want to check first.
243
+
244
+ `analyze({ ..., includeChart: true })` adds `chart` (both posterior densities) and `liftChart` (the exact density of the relative lift, peak scaled to 1), which is what the web tool plots.
245
+
246
+ The primitives behind it are exported too, for anything that needs the same math without a full analysis:
247
+
248
+ | Export | What it does |
249
+ |---|---|
250
+ | `betaPdf(x, alpha, beta)`, `logBeta`, `logGamma` | Beta density and its log-gamma building blocks |
251
+ | `probabilityBBeatsA(alphaA, betaA, alphaB, betaB)` | P(B > A) for two Beta posteriors: exact closed form for whole-number alphas, numerical integration otherwise |
252
+ | `regularizedIncompleteBeta(x, a, b)` | The Beta(a, b) CDF at x |
253
+ | `integratedExpectedLoss(alphaA, betaA, alphaB, betaB)` | Expected loss of shipping each variant by numerical integration, no sampling |
254
+ | `liftDensity(alphaA, betaA, alphaB, betaB, sortedLifts)` | Exact relative-lift density over the range the sorted lifts span |
255
+
256
+ This package is the source of truth for the math on joebrundage.com: the [calculator](https://www.joebrundage.com/tools/bayesian-ab-test) and the [primer](https://www.joebrundage.com/tools/bayesian-ab-testing-primer) both import it. Change the math here, publish, and bump the site's dependency.
257
+
232
258
  ## Method
233
259
 
234
- - Each conversion rate gets a Beta(1,1) prior. Posteriors are Beta(1 + conversions, 1 + non-conversions).
235
- - P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html).
260
+ - Each conversion rate gets a Beta(α, β) prior, Beta(1,1) unless you choose another. Posteriors are Beta(α + conversions, β + non-conversions).
261
+ - P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html) when α is a whole number. For a fractional α it is integrated numerically (tanh-sinh quadrature over the incomplete beta function), accurate to about 1e-10. P(B is harmful) is its exact complement.
236
262
  - Lift intervals, threshold probabilities and expected loss come from 12,000 seeded Monte Carlo draws.
237
263
  - The meaningful effect is relative and symmetric: `--min-effect 10` means B must be 10% better than A to count as a win or 10% worse to count as a loss.
238
264
  - `project` samples true rates from the exact posterior, then uses a normal approximation to the posterior when re-checking the stopping rule on each simulated day. Day zero is checked exactly.
package/dist/cli.js CHANGED
@@ -119,11 +119,80 @@ function probabilityRightBeatsLeftDirect(alphaLeft, betaLeft, alphaRight, betaRi
119
119
  return Math.exp(maxLog) * scaledSum;
120
120
  }
121
121
  function probabilityBBeatsA(alphaA, betaA, alphaB, betaB) {
122
+ if (!Number.isInteger(alphaA) || !Number.isInteger(alphaB)) {
123
+ return probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB);
124
+ }
122
125
  if (alphaB <= alphaA) {
123
126
  return probabilityRightBeatsLeftDirect(alphaA, betaA, alphaB, betaB);
124
127
  }
125
128
  return 1 - probabilityRightBeatsLeftDirect(alphaB, betaB, alphaA, betaA);
126
129
  }
130
+ function betaContinuedFraction(x, a, b) {
131
+ const TINY = 1e-300;
132
+ const EPSILON = 1e-15;
133
+ const MAX_ITERATIONS = 1e5;
134
+ const qab = a + b;
135
+ const qap = a + 1;
136
+ const qam = a - 1;
137
+ let c = 1;
138
+ let d = 1 - qab * x / qap;
139
+ if (Math.abs(d) < TINY) d = TINY;
140
+ d = 1 / d;
141
+ let h = d;
142
+ for (let m = 1; m <= MAX_ITERATIONS; m += 1) {
143
+ const m2 = 2 * m;
144
+ let aa = m * (b - m) * x / ((qam + m2) * (a + m2));
145
+ d = 1 + aa * d;
146
+ if (Math.abs(d) < TINY) d = TINY;
147
+ c = 1 + aa / c;
148
+ if (Math.abs(c) < TINY) c = TINY;
149
+ d = 1 / d;
150
+ h *= d * c;
151
+ aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
152
+ d = 1 + aa * d;
153
+ if (Math.abs(d) < TINY) d = TINY;
154
+ c = 1 + aa / c;
155
+ if (Math.abs(c) < TINY) c = TINY;
156
+ d = 1 / d;
157
+ const delta = d * c;
158
+ h *= delta;
159
+ if (Math.abs(delta - 1) < EPSILON) break;
160
+ }
161
+ return h;
162
+ }
163
+ function regularizedIncompleteBeta(x, a, b) {
164
+ if (x <= 0) return 0;
165
+ if (x >= 1) return 1;
166
+ if (x > (a + 1) / (a + b + 2)) return 1 - regularizedIncompleteBeta(1 - x, b, a);
167
+ const logFront = a * Math.log(x) + b * Math.log1p(-x) - logBeta(a, b);
168
+ return Math.exp(logFront) * betaContinuedFraction(x, a, b) / a;
169
+ }
170
+ function probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB) {
171
+ const total = alphaA + betaA;
172
+ const meanA = alphaA / total;
173
+ const sdA = Math.sqrt(alphaA * betaA / (total * total * (total + 1)));
174
+ const low = Math.max(0, meanA - 40 * sdA);
175
+ const high = Math.min(1, meanA + 40 * sdA);
176
+ const center = (low + high) / 2;
177
+ const halfWidth = (high - low) / 2;
178
+ const logNormalizer = logBeta(alphaA, betaA);
179
+ const STEP = 1 / 128;
180
+ const T_MAX = 3.5;
181
+ let mass = 0;
182
+ let weighted = 0;
183
+ for (let t = -T_MAX; t <= T_MAX + 1e-12; t += STEP) {
184
+ const inner = Math.PI / 2 * Math.sinh(t);
185
+ const x = center + halfWidth * Math.tanh(inner);
186
+ if (x <= 0 || x >= 1) continue;
187
+ const coshInner = Math.cosh(inner);
188
+ const weight = Math.PI / 2 * Math.cosh(t) / (coshInner * coshInner);
189
+ const density = Math.exp((alphaA - 1) * Math.log(x) + (betaA - 1) * Math.log1p(-x) - logNormalizer);
190
+ if (!Number.isFinite(density)) continue;
191
+ mass += weight * density;
192
+ weighted += weight * density * (1 - regularizedIncompleteBeta(x, alphaB, betaB));
193
+ }
194
+ return mass > 0 ? weighted / mass : 0.5;
195
+ }
127
196
  function inverseNormalCdf(p) {
128
197
  const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
129
198
  const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
@@ -158,6 +227,50 @@ function round(value, decimals = 6) {
158
227
  return Math.round(value * factor) / factor;
159
228
  }
160
229
 
230
+ // src/core/lift.ts
231
+ var LIFT_DENSITY_POINTS = 160;
232
+ var LIFT_INTEGRATION_STEPS = 400;
233
+ function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
234
+ const lower = quantile(sortedRelativeLifts, 2e-3);
235
+ const upper = quantile(sortedRelativeLifts, 0.998);
236
+ const span = upper - lower;
237
+ if (!Number.isFinite(span) || span <= 0) {
238
+ return [
239
+ { relativeLift: lower, density: 0 },
240
+ { relativeLift: lower, density: 1 },
241
+ { relativeLift: lower, density: 0 }
242
+ ];
243
+ }
244
+ const totalA = alphaA + betaA;
245
+ const meanA = alphaA / totalA;
246
+ const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
247
+ const aLow = Math.max(1e-9, meanA - 10 * sdA);
248
+ const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
249
+ const lifts = Array.from(
250
+ { length: LIFT_DENSITY_POINTS },
251
+ (_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
252
+ );
253
+ const densities = lifts.map((relativeLift) => {
254
+ const ratio = 1 + relativeLift;
255
+ if (ratio <= 0) return 0;
256
+ const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
257
+ if (aHigh <= aLow) return 0;
258
+ const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
259
+ let total = 0;
260
+ for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
261
+ const a = aLow + step * index;
262
+ const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
263
+ total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
264
+ }
265
+ return total * step / 3;
266
+ });
267
+ const peak = Math.max(...densities, Number.MIN_VALUE);
268
+ return lifts.map((relativeLift, index) => ({
269
+ relativeLift,
270
+ density: densities[index] / peak
271
+ }));
272
+ }
273
+
161
274
  // src/core/plan.ts
162
275
  var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
163
276
  function requiredSamplePerVariant({
@@ -204,6 +317,10 @@ function shareUrl(inputs) {
204
317
  thresholdPercent: formatPercentParam(inputs.threshold),
205
318
  meaningfulLiftPercent: formatPercentParam(inputs.minEffect)
206
319
  });
320
+ if (inputs.prior && !(inputs.prior[0] === 1 && inputs.prior[1] === 1)) {
321
+ params.set("priorAlpha", String(inputs.prior[0]));
322
+ params.set("priorBeta", String(inputs.prior[1]));
323
+ }
207
324
  return `${WEB_TOOL_URL}?${params.toString()}`;
208
325
  }
209
326
 
@@ -215,13 +332,20 @@ var PRIOR_ALPHA = 1;
215
332
  var PRIOR_BETA = 1;
216
333
  var POSTERIOR_CHART_POINTS = 96;
217
334
  var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
218
- var WAITING_VALUE_SIMULATIONS = 1500;
335
+ var WAITING_VALUE_SIMULATIONS = 6e3;
219
336
  var DECISION_LABELS = {
220
337
  ship_b: "Ship Variant B",
221
338
  keep_a: "Keep Variant A",
222
339
  continue: "Continue test",
223
340
  inconclusive: "Inconclusive"
224
341
  };
342
+ function validatePrior(prior) {
343
+ const [alpha, beta] = prior;
344
+ if (!(Number.isFinite(alpha) && alpha > 0 && Number.isFinite(beta) && beta > 0)) {
345
+ throw new RangeError(`Prior parameters must be finite and greater than 0, got Beta(${alpha}, ${beta}).`);
346
+ }
347
+ return [alpha, beta];
348
+ }
225
349
  function deriveSeed(...parts) {
226
350
  let hash = 2166136261;
227
351
  for (const part of parts) {
@@ -276,8 +400,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
276
400
  });
277
401
  }
278
402
  function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
279
- const random = makeDeterministicRandom(alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19);
403
+ const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
280
404
  return WAITING_SCENARIOS.map((extraPerVariant) => {
405
+ const random = makeDeterministicRandom(seed);
281
406
  let regretSum = 0;
282
407
  for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
283
408
  const trueRateA = sampleBeta(alphaA, betaA, random);
@@ -303,12 +428,14 @@ function analyze({
303
428
  minEffect = DEFAULT_MIN_EFFECT,
304
429
  samples = DEFAULT_SAMPLES,
305
430
  seed = null,
306
- includeChart = false
431
+ includeChart = false,
432
+ prior = [PRIOR_ALPHA, PRIOR_BETA]
307
433
  }) {
308
- const alphaA = PRIOR_ALPHA + conversionsA;
309
- const betaA = PRIOR_BETA + (visitorsA - conversionsA);
310
- const alphaB = PRIOR_ALPHA + conversionsB;
311
- const betaB = PRIOR_BETA + (visitorsB - conversionsB);
434
+ const [priorAlpha, priorBeta] = validatePrior(prior);
435
+ const alphaA = priorAlpha + conversionsA;
436
+ const betaA = priorBeta + (visitorsA - conversionsA);
437
+ const alphaB = priorAlpha + conversionsB;
438
+ const betaB = priorBeta + (visitorsB - conversionsB);
312
439
  const rateA = conversionsA / visitorsA;
313
440
  const rateB = conversionsB / visitorsB;
314
441
  const posteriorMeanA = alphaA / (alphaA + betaA);
@@ -343,7 +470,7 @@ function analyze({
343
470
  conversionsB,
344
471
  threshold,
345
472
  minEffect,
346
- prior: [PRIOR_ALPHA, PRIOR_BETA],
473
+ prior: [priorAlpha, priorBeta],
347
474
  samples,
348
475
  seed: resolvedSeed
349
476
  },
@@ -361,7 +488,9 @@ function analyze({
361
488
  aBeatsB: 1 - pBBeatsA,
362
489
  meaningfulLift: probabilityMeaningfulLift,
363
490
  meaningfulHarm: probabilityMeaningfulHarm,
364
- harmful: share((lift) => lift < 0),
491
+ /* The exact complement of the closed-form win probability. Estimating it
492
+ from the samples left the two halves summing to slightly less than 1. */
493
+ harmful: 1 - pBBeatsA,
365
494
  liftAbove1pct: share((lift) => lift > 0.01),
366
495
  liftAbove5pct: share((lift) => lift > 0.05),
367
496
  harmBeyond5pct: share((lift) => lift < -0.05),
@@ -373,7 +502,9 @@ function analyze({
373
502
  absoluteDiff: posteriorMeanB - posteriorMeanA,
374
503
  absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
375
504
  observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
376
- observedAbsoluteDiff: rateB - rateA
505
+ observedAbsoluteDiff: rateB - rateA,
506
+ relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
507
+ relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
377
508
  },
378
509
  expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
379
510
  variants: {
@@ -387,12 +518,13 @@ function analyze({
387
518
  },
388
519
  waiting: waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow),
389
520
  frequentist: { pValueTwoSided: twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) },
390
- shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect })
521
+ shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] })
391
522
  };
392
523
  if (includeChart) {
393
524
  const sortedA = [...samplesA].sort((left, right) => left - right);
394
525
  const sortedB = [...samplesB].sort((left, right) => left - right);
395
526
  result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
527
+ result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
396
528
  }
397
529
  return result;
398
530
  }
@@ -474,9 +606,11 @@ function project({
474
606
  stopRule = "lift",
475
607
  maxLoss = DEFAULT_MAX_LOSS,
476
608
  simulations = DEFAULT_SIMULATIONS,
477
- seed = null
609
+ seed = null,
610
+ prior = [PRIOR_ALPHA, PRIOR_BETA]
478
611
  }) {
479
- const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
612
+ const [priorAlpha, priorBeta] = validatePrior(prior);
613
+ const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
480
614
  const resolvedSeed = seed ?? deriveSeed(visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, ["lift", "superiority", "loss"].indexOf(stopRule), simulations);
481
615
  const inputs = {
482
616
  visitorsA,
@@ -485,7 +619,7 @@ function project({
485
619
  conversionsB,
486
620
  threshold,
487
621
  minEffect,
488
- prior: [PRIOR_ALPHA, PRIOR_BETA],
622
+ prior: [priorAlpha, priorBeta],
489
623
  dailyVisitors,
490
624
  horizonDays,
491
625
  stopRule,
@@ -493,7 +627,7 @@ function project({
493
627
  simulations,
494
628
  seed: resolvedSeed
495
629
  };
496
- const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
630
+ const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
497
631
  const diagnostic = stopRule === "lift" && liftNearThreshold(current, minEffect) ? "lift_near_threshold" : null;
498
632
  const decisionNow = decisionNowFrom(current, stopRule, threshold, maxLoss);
499
633
  if (decisionNow) {
@@ -510,10 +644,10 @@ function project({
510
644
  shareUrl: link
511
645
  };
512
646
  }
513
- const alphaA0 = PRIOR_ALPHA + conversionsA;
514
- const betaA0 = PRIOR_BETA + (visitorsA - conversionsA);
515
- const alphaB0 = PRIOR_ALPHA + conversionsB;
516
- const betaB0 = PRIOR_BETA + (visitorsB - conversionsB);
647
+ const alphaA0 = priorAlpha + conversionsA;
648
+ const betaA0 = priorBeta + (visitorsA - conversionsA);
649
+ const alphaB0 = priorAlpha + conversionsB;
650
+ const betaB0 = priorBeta + (visitorsB - conversionsB);
517
651
  const random = makeDeterministicRandom(resolvedSeed);
518
652
  const decidedByDay = new Array(horizonDays + 1).fill(0);
519
653
  const decisionDays = [];
@@ -593,6 +727,8 @@ var OPTIONS = {
593
727
  "rate-b": { type: "string" },
594
728
  threshold: { type: "string" },
595
729
  "min-effect": { type: "string" },
730
+ "prior-alpha": { type: "string" },
731
+ "prior-beta": { type: "string" },
596
732
  "daily-visitors": { type: "string" },
597
733
  horizon: { type: "string" },
598
734
  "stop-rule": { type: "string" },
@@ -683,10 +819,19 @@ function parseMinEffect(values) {
683
819
  if (minEffect < 0 || minEffect >= 1) throw new CliError("--min-effect must be a percent between 0 and 100.", { field: "min-effect" });
684
820
  return minEffect;
685
821
  }
822
+ function parsePriorParameter(flag, value, fallback) {
823
+ if (value === void 0) return fallback;
824
+ const parsed = parseNumber(flag, value);
825
+ if (parsed <= 0) throw new CliError(`--${flag} must be greater than 0, got "${value}".`, { field: flag });
826
+ return parsed;
827
+ }
828
+ function parsePrior(values) {
829
+ return [parsePriorParameter("prior-alpha", values["prior-alpha"], PRIOR_ALPHA), parsePriorParameter("prior-beta", values["prior-beta"], PRIOR_BETA)];
830
+ }
686
831
  function parseAnalyzeArgs(values) {
687
832
  const samples = values.samples === void 0 ? DEFAULT_SAMPLES : parseCount("samples", values.samples, { min: 100 });
688
833
  const seed = values.seed === void 0 ? null : parseCount("seed", values.seed);
689
- return { ...parseVariants(values), threshold: parseThreshold(values), minEffect: parseMinEffect(values), samples, seed };
834
+ return { ...parseVariants(values), threshold: parseThreshold(values), minEffect: parseMinEffect(values), prior: parsePrior(values), samples, seed };
690
835
  }
691
836
  function parseStopRule(value) {
692
837
  if (value === void 0) return "lift";
@@ -762,7 +907,8 @@ function helpText(version) {
762
907
  return `bayes-ab ${version} Bayesian A/B test analysis for humans and agents
763
908
 
764
909
  Compares two variants of a binary outcome (signup, click, purchase) using
765
- Beta(1,1) posteriors on each conversion rate. Reports the probability that B
910
+ Beta posteriors on each conversion rate, from a flat Beta(1,1) prior unless
911
+ you choose another. Reports the probability that B
766
912
  beats A, the probability the lift clears a meaningful threshold, expected
767
913
  loss from shipping either variant, and a forward projection of how long the
768
914
  test needs to run. Same math as ${WEB_TOOL_URL}
@@ -789,6 +935,12 @@ INPUTS (analyze, project)
789
935
  --min-effect PCT Meaningful relative effect, symmetric: with 10, B must be
790
936
  10% better than A to count as a win or 10% worse to count
791
937
  as a loss. Relative, not percentage points. default 10
938
+ --prior-alpha N, --prior-beta N Beta(alpha, beta) prior on each variant's conversion rate,
939
+ the same for both. Any positive number. Reads as alpha
940
+ conversions in alpha + beta visitors of prior evidence,
941
+ with prior mean alpha / (alpha + beta).
942
+ Beta(1,1) is flat; Beta(0.5,0.5) is Jeffreys; Beta(55,945)
943
+ says "about 5.5%, worth 1,000 visitors". default 1, 1
792
944
  Whole numbers may include commas or underscores: 12,000 and 12_000 both work.
793
945
 
794
946
  PROJECTION (project)
@@ -819,7 +971,7 @@ OUTPUT
819
971
  so piping or calling from an agent gets JSON automatically.
820
972
  --pretty Indent the JSON.
821
973
  --brief One-line human summary (analyze and project).
822
- --include-chart Add 96 posterior density points to analyze's JSON.
974
+ --include-chart Add posterior and lift density curves to analyze's JSON.
823
975
  --samples N Monte Carlo samples for intervals and loss. default 12000
824
976
  --seed N RNG seed. Defaults to a value derived from the inputs, so identical
825
977
  inputs always give identical output.
@@ -884,6 +1036,10 @@ function unsignedPoints(fraction2, digits = 4) {
884
1036
  function count(value) {
885
1037
  return Math.round(value).toLocaleString("en-US");
886
1038
  }
1039
+ function betaParameter(value) {
1040
+ if (Number.isInteger(value)) return count(value);
1041
+ return value.toLocaleString("en-US", { maximumFractionDigits: 3 });
1042
+ }
887
1043
  function missedConversions(regretRate, visitors = 1e4) {
888
1044
  const missed = regretRate * visitors;
889
1045
  const digits = missed < 1 ? 2 : missed < 10 ? 1 : 0;
@@ -937,6 +1093,8 @@ function renderAnalyzeBrief(result, derivedFrom2) {
937
1093
  `lift ${signedPercent(result.effect.relativeLift)} (CI ${signedPercent(lower)} to ${signedPercent(upper)})`,
938
1094
  `loss if ship B ${unsignedPoints(result.expectedLoss.shipB)}`
939
1095
  ];
1096
+ const [priorAlpha, priorBeta] = result.inputs.prior;
1097
+ if (priorAlpha !== 1 || priorBeta !== 1) parts.push(`Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) prior`);
940
1098
  const note = derivedNote(result, derivedFrom2);
941
1099
  return parts.join(" \xB7 ") + (note ? ` ${note}` : "");
942
1100
  }
@@ -948,7 +1106,7 @@ function renderAnalyze(result, derivedFrom2) {
948
1106
  const lines = [];
949
1107
  const note = derivedNote(result, derivedFrom2);
950
1108
  lines.push(
951
- `Bayesian A/B test Beta(${inputs.prior[0]},${inputs.prior[1]}) prior \xB7 decision threshold ${percent(inputs.threshold, 0)} \xB7 meaningful effect \xB1${minEffect} relative`
1109
+ `Bayesian A/B test Beta(${betaParameter(inputs.prior[0])},${betaParameter(inputs.prior[1])}) prior \xB7 decision threshold ${percent(inputs.threshold, 0)} \xB7 meaningful effect \xB1${minEffect} relative`
952
1110
  );
953
1111
  if (note) lines.push(` ${note}`);
954
1112
  lines.push("");
@@ -974,7 +1132,7 @@ function renderAnalyze(result, derivedFrom2) {
974
1132
  ["Conversions", count(variants.A.conversions), count(variants.B.conversions)],
975
1133
  ["Observed rate", percent(variants.A.rate), percent(variants.B.rate)],
976
1134
  ["Posterior mean", percent(variants.A.posteriorMean, 3), percent(variants.B.posteriorMean, 3)],
977
- ["Posterior", `Beta(${count(variants.A.posterior[0])}, ${count(variants.A.posterior[1])})`, `Beta(${count(variants.B.posterior[0])}, ${count(variants.B.posterior[1])})`]
1135
+ ["Posterior", `Beta(${betaParameter(variants.A.posterior[0])}, ${betaParameter(variants.A.posterior[1])})`, `Beta(${betaParameter(variants.B.posterior[0])}, ${betaParameter(variants.B.posterior[1])})`]
978
1136
  ])
979
1137
  );
980
1138
  lines.push("");
@@ -1112,6 +1270,8 @@ function renderProjectBrief(result) {
1112
1270
  result.daysToDecision ? `median day ${result.daysToDecision.p50} if it decides (p90 day ${result.daysToDecision.p90})` : "no simulated future decided",
1113
1271
  `${count(dailyVisitors)}/day per variant`
1114
1272
  ];
1273
+ const [priorAlpha, priorBeta] = result.inputs.prior;
1274
+ if (priorAlpha !== 1 || priorBeta !== 1) parts.push(`Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) prior`);
1115
1275
  return parts.join(" \xB7 ");
1116
1276
  }
1117
1277
  function renderProject(result) {
@@ -1121,6 +1281,8 @@ function renderProject(result) {
1121
1281
  `Forward projection ${count(inputs.dailyVisitors)} visitors/day per variant \xB7 ${inputs.horizonDays}-day horizon \xB7 ${count(inputs.simulations)} simulated futures`
1122
1282
  );
1123
1283
  lines.push(`Stopping rule ${describeRule(result)}`);
1284
+ const [priorAlpha, priorBeta] = inputs.prior;
1285
+ if (priorAlpha !== 1 || priorBeta !== 1) lines.push(`Prior Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) on each variant`);
1124
1286
  lines.push("");
1125
1287
  if (result.alreadyDecided && result.decisionNow) {
1126
1288
  lines.push(`Already decided ${decisionLabel(result.decisionNow)}`);
@@ -1186,7 +1348,7 @@ var variantInputs = {
1186
1348
  conversionsB: integer,
1187
1349
  threshold: fraction,
1188
1350
  minEffect: fraction,
1189
- prior: pair(number)
1351
+ prior: { ...pair({ type: "number", exclusiveMinimum: 0 }), description: "The Beta(alpha, beta) prior applied to each variant. [1, 1] unless --prior-alpha or --prior-beta was given." }
1190
1352
  };
1191
1353
  var derivedFrom = {
1192
1354
  ...object({ rateA: fraction, rateB: fraction }, []),
@@ -1217,7 +1379,7 @@ var ANALYZE_SCHEMA = {
1217
1379
  aBeatsB: fraction,
1218
1380
  meaningfulLift: { ...fraction, description: "P(relative lift > minEffect)" },
1219
1381
  meaningfulHarm: { ...fraction, description: "P(relative lift < -minEffect)" },
1220
- harmful: { ...fraction, description: "P(relative lift < 0)" },
1382
+ harmful: { ...fraction, description: "P(B < A), the exact complement of bBeatsA" },
1221
1383
  liftAbove1pct: fraction,
1222
1384
  liftAbove5pct: fraction,
1223
1385
  harmBeyond5pct: fraction,
@@ -1229,7 +1391,9 @@ var ANALYZE_SCHEMA = {
1229
1391
  absoluteDiff: number,
1230
1392
  absoluteDiffCI95: pair(number),
1231
1393
  observedRelativeLift: { type: ["number", "null"] },
1232
- observedAbsoluteDiff: number
1394
+ observedAbsoluteDiff: number,
1395
+ relativeLiftMedian: number,
1396
+ relativeLiftIQR: pair(number)
1233
1397
  }),
1234
1398
  expectedLoss: object({ shipA: fraction, shipB: fraction }),
1235
1399
  variants: object({
@@ -1251,6 +1415,11 @@ var ANALYZE_SCHEMA = {
1251
1415
  description: "Only with --include-chart.",
1252
1416
  items: object({ rate: fraction, densityA: number, densityB: number })
1253
1417
  },
1418
+ liftChart: {
1419
+ type: "array",
1420
+ description: "Only with --include-chart. Exact density of the relative lift, peak scaled to 1.",
1421
+ items: object({ relativeLift: number, density: number })
1422
+ },
1254
1423
  shareUrl: { type: "string", format: "uri" }
1255
1424
  }, [
1256
1425
  "tool",
@@ -1343,7 +1512,7 @@ var SCHEMAS = {
1343
1512
  };
1344
1513
 
1345
1514
  // src/cli/main.ts
1346
- var VERSION = true ? "0.1.0" : "dev";
1515
+ var VERSION = true ? "0.3.0" : "dev";
1347
1516
  var JSON_DECIMALS = 6;
1348
1517
  function roundDeep(value) {
1349
1518
  if (typeof value === "number") return round(value, JSON_DECIMALS);
@@ -1367,7 +1536,7 @@ function envelope2(command, result, derivedFrom2) {
1367
1536
  return { tool: "bayes-ab", version: VERSION, command, ...result, inputs };
1368
1537
  }
1369
1538
  function stripChart(result) {
1370
- const { chart, ...rest } = result;
1539
+ const { chart, liftChart, ...rest } = result;
1371
1540
  return rest;
1372
1541
  }
1373
1542
  function run(argv, io) {
@@ -1,3 +1,4 @@
1
+ import { type LiftDensityPoint } from "./lift.ts";
1
2
  export declare const DEFAULT_THRESHOLD = 0.95;
2
3
  export declare const DEFAULT_MIN_EFFECT = 0.1;
3
4
  export declare const DEFAULT_SAMPLES = 12000;
@@ -18,6 +19,8 @@ export type AnalyzeInputs = {
18
19
  samples?: number;
19
20
  seed?: number | null;
20
21
  includeChart?: boolean;
22
+ /** Beta(alpha, beta) prior applied to both variants. Defaults to the flat Beta(1, 1). */
23
+ prior?: [number, number];
21
24
  };
22
25
  export type VariantSummary = {
23
26
  visitors: number;
@@ -75,6 +78,8 @@ export type AnalyzeResult = {
75
78
  absoluteDiffCI95: [number, number];
76
79
  observedRelativeLift: number | null;
77
80
  observedAbsoluteDiff: number;
81
+ relativeLiftMedian: number;
82
+ relativeLiftIQR: [number, number];
78
83
  };
79
84
  expectedLoss: {
80
85
  shipA: number;
@@ -94,8 +99,12 @@ export type AnalyzeResult = {
94
99
  pValueTwoSided: number;
95
100
  };
96
101
  chart?: ChartPoint[];
102
+ liftChart?: LiftDensityPoint[];
97
103
  shareUrl: string;
98
104
  };
105
+ /** Throws a RangeError unless both prior parameters are finite and positive. */
106
+ export declare function validatePrior(prior: [number, number]): [number, number];
107
+ export declare function isDefaultPrior([alpha, beta]: [number, number]): boolean;
99
108
  export declare function deriveSeed(...parts: number[]): number;
100
109
  export declare function decisionStatus({ probabilityMeaningfulLift, probabilityMeaningfulHarm, threshold, evidence, }: {
101
110
  probabilityMeaningfulLift: number;
@@ -103,4 +112,4 @@ export declare function decisionStatus({ probabilityMeaningfulLift, probabilityM
103
112
  threshold: number;
104
113
  evidence: EvidenceStrength;
105
114
  }): DecisionStatus;
106
- export declare function analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, samples, seed, includeChart, }: AnalyzeInputs): AnalyzeResult;
115
+ export declare function analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, samples, seed, includeChart, prior, }: AnalyzeInputs): AnalyzeResult;
@@ -0,0 +1,15 @@
1
+ export type LiftDensityPoint = {
2
+ relativeLift: number;
3
+ /** Scaled so the peak is 1. */
4
+ density: number;
5
+ };
6
+ /**
7
+ * Density of B's relative lift over A, computed exactly rather than
8
+ * histogrammed from the Monte Carlo samples. Writing B = A(1 + r), the ratio
9
+ * density is f(r) = ∫ a · f_A(a) · f_B(a(1+r)) da, which is one Simpson
10
+ * integral per grid point. A histogram of 12k draws carries ~10% sampling
11
+ * noise per bin, and on a wide posterior that reads as a jagged curve.
12
+ *
13
+ * The sorted Monte Carlo lifts only set the plotted range (0.2% to 99.8%).
14
+ */
15
+ export declare function liftDensity(alphaA: number, betaA: number, alphaB: number, betaB: number, sortedRelativeLifts: number[]): LiftDensityPoint[];
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Expected loss of each choice by direct numerical integration over a 600-step
3
+ * grid, with no sampling. Deterministic and smooth in its inputs, so it suits
4
+ * curves that sweep sample size; `analyze` uses Monte Carlo instead because it
5
+ * shares those draws with the lift interval and threshold probabilities.
6
+ *
7
+ * Losses are absolute conversion-rate differences (0.001 = 0.1 pp):
8
+ * shipA = E[max(B − A, 0)], shipB = E[max(A − B, 0)].
9
+ */
10
+ export declare function integratedExpectedLoss(alphaA: number, betaA: number, alphaB: number, betaB: number): {
11
+ shipA: number;
12
+ shipB: number;
13
+ };
@@ -10,8 +10,16 @@ export declare function sampleBeta(alpha: number, beta: number, random: Random):
10
10
  export declare function sampleBinomialApprox(trials: number, probability: number, random: Random): number;
11
11
  export declare function quantile(sortedValues: number[], probability: number): number;
12
12
  export declare function mean(values: number[]): number;
13
- /** Exact P(B > A) for independent Beta posteriors, via Evan Miller's closed form. */
13
+ /**
14
+ * P(B > A) for independent Beta posteriors. Exact via Evan Miller's closed form
15
+ * when the alphas are whole numbers, which they are under any whole-number
16
+ * prior. The closed form sums over 0..alpha-1, so a fractional alpha (a
17
+ * Jeffreys Beta(0.5, 0.5) prior, say) falls back to numerical integration,
18
+ * accurate to roughly 1e-10.
19
+ */
14
20
  export declare function probabilityBBeatsA(alphaA: number, betaA: number, alphaB: number, betaB: number): number;
21
+ /** Regularized incomplete beta function I_x(a, b): the Beta(a, b) CDF at x. */
22
+ export declare function regularizedIncompleteBeta(x: number, a: number, b: number): number;
15
23
  export declare function inverseNormalCdf(p: number): number;
16
24
  export declare function erf(x: number): number;
17
25
  export declare function normalCdf(x: number): number;
@@ -16,6 +16,8 @@ export type ProjectInputs = {
16
16
  maxLoss?: number;
17
17
  simulations?: number;
18
18
  seed?: number | null;
19
+ /** Beta(alpha, beta) prior applied to both variants. Defaults to the flat Beta(1, 1). */
20
+ prior?: [number, number];
19
21
  };
20
22
  export type ProjectResult = {
21
23
  inputs: {
@@ -55,4 +57,4 @@ export type ProjectResult = {
55
57
  approximation: "normal";
56
58
  shareUrl: string;
57
59
  };
58
- export declare function project({ visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, threshold, minEffect, stopRule, maxLoss, simulations, seed, }: ProjectInputs): ProjectResult;
60
+ export declare function project({ visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, threshold, minEffect, stopRule, maxLoss, simulations, seed, prior, }: ProjectInputs): ProjectResult;
@@ -6,5 +6,6 @@ export type ShareInputs = {
6
6
  conversionsB: number;
7
7
  threshold: number;
8
8
  minEffect: number;
9
+ prior?: [number, number];
9
10
  };
10
11
  export declare function shareUrl(inputs: ShareInputs): string;
package/dist/index.d.ts CHANGED
@@ -1,8 +1,11 @@
1
- export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD } from "./core/analyze.ts";
1
+ export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD, PRIOR_ALPHA, PRIOR_BETA, isDefaultPrior, validatePrior } from "./core/analyze.ts";
2
2
  export type { AnalyzeInputs, AnalyzeResult, ChartPoint, DecisionStatus, EvidenceStrength, Maturity, Variant, VariantSummary, WaitingScenario, } from "./core/analyze.ts";
3
3
  export { project, DEFAULT_HORIZON_DAYS, DEFAULT_MAX_LOSS, DEFAULT_SIMULATIONS } from "./core/project.ts";
4
4
  export type { ProjectInputs, ProjectResult, ProjectedDecision, StopRule } from "./core/project.ts";
5
5
  export { plan, requiredSamplePerVariant } from "./core/plan.ts";
6
6
  export type { PlanInputs, PlanResult, PlanRow } from "./core/plan.ts";
7
7
  export { shareUrl, WEB_TOOL_URL } from "./core/share.ts";
8
- export { probabilityBBeatsA } from "./core/math.ts";
8
+ export { betaPdf, logBeta, logGamma, probabilityBBeatsA, regularizedIncompleteBeta } from "./core/math.ts";
9
+ export { liftDensity } from "./core/lift.ts";
10
+ export type { LiftDensityPoint } from "./core/lift.ts";
11
+ export { integratedExpectedLoss } from "./core/loss.ts";
package/dist/index.js CHANGED
@@ -117,11 +117,80 @@ function probabilityRightBeatsLeftDirect(alphaLeft, betaLeft, alphaRight, betaRi
117
117
  return Math.exp(maxLog) * scaledSum;
118
118
  }
119
119
  function probabilityBBeatsA(alphaA, betaA, alphaB, betaB) {
120
+ if (!Number.isInteger(alphaA) || !Number.isInteger(alphaB)) {
121
+ return probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB);
122
+ }
120
123
  if (alphaB <= alphaA) {
121
124
  return probabilityRightBeatsLeftDirect(alphaA, betaA, alphaB, betaB);
122
125
  }
123
126
  return 1 - probabilityRightBeatsLeftDirect(alphaB, betaB, alphaA, betaA);
124
127
  }
128
+ function betaContinuedFraction(x, a, b) {
129
+ const TINY = 1e-300;
130
+ const EPSILON = 1e-15;
131
+ const MAX_ITERATIONS = 1e5;
132
+ const qab = a + b;
133
+ const qap = a + 1;
134
+ const qam = a - 1;
135
+ let c = 1;
136
+ let d = 1 - qab * x / qap;
137
+ if (Math.abs(d) < TINY) d = TINY;
138
+ d = 1 / d;
139
+ let h = d;
140
+ for (let m = 1; m <= MAX_ITERATIONS; m += 1) {
141
+ const m2 = 2 * m;
142
+ let aa = m * (b - m) * x / ((qam + m2) * (a + m2));
143
+ d = 1 + aa * d;
144
+ if (Math.abs(d) < TINY) d = TINY;
145
+ c = 1 + aa / c;
146
+ if (Math.abs(c) < TINY) c = TINY;
147
+ d = 1 / d;
148
+ h *= d * c;
149
+ aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
150
+ d = 1 + aa * d;
151
+ if (Math.abs(d) < TINY) d = TINY;
152
+ c = 1 + aa / c;
153
+ if (Math.abs(c) < TINY) c = TINY;
154
+ d = 1 / d;
155
+ const delta = d * c;
156
+ h *= delta;
157
+ if (Math.abs(delta - 1) < EPSILON) break;
158
+ }
159
+ return h;
160
+ }
161
+ function regularizedIncompleteBeta(x, a, b) {
162
+ if (x <= 0) return 0;
163
+ if (x >= 1) return 1;
164
+ if (x > (a + 1) / (a + b + 2)) return 1 - regularizedIncompleteBeta(1 - x, b, a);
165
+ const logFront = a * Math.log(x) + b * Math.log1p(-x) - logBeta(a, b);
166
+ return Math.exp(logFront) * betaContinuedFraction(x, a, b) / a;
167
+ }
168
+ function probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB) {
169
+ const total = alphaA + betaA;
170
+ const meanA = alphaA / total;
171
+ const sdA = Math.sqrt(alphaA * betaA / (total * total * (total + 1)));
172
+ const low = Math.max(0, meanA - 40 * sdA);
173
+ const high = Math.min(1, meanA + 40 * sdA);
174
+ const center = (low + high) / 2;
175
+ const halfWidth = (high - low) / 2;
176
+ const logNormalizer = logBeta(alphaA, betaA);
177
+ const STEP = 1 / 128;
178
+ const T_MAX = 3.5;
179
+ let mass = 0;
180
+ let weighted = 0;
181
+ for (let t = -T_MAX; t <= T_MAX + 1e-12; t += STEP) {
182
+ const inner = Math.PI / 2 * Math.sinh(t);
183
+ const x = center + halfWidth * Math.tanh(inner);
184
+ if (x <= 0 || x >= 1) continue;
185
+ const coshInner = Math.cosh(inner);
186
+ const weight = Math.PI / 2 * Math.cosh(t) / (coshInner * coshInner);
187
+ const density = Math.exp((alphaA - 1) * Math.log(x) + (betaA - 1) * Math.log1p(-x) - logNormalizer);
188
+ if (!Number.isFinite(density)) continue;
189
+ mass += weight * density;
190
+ weighted += weight * density * (1 - regularizedIncompleteBeta(x, alphaB, betaB));
191
+ }
192
+ return mass > 0 ? weighted / mass : 0.5;
193
+ }
125
194
  function inverseNormalCdf(p) {
126
195
  const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
127
196
  const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
@@ -151,6 +220,50 @@ function normalPdf(x) {
151
220
  return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
152
221
  }
153
222
 
223
+ // src/core/lift.ts
224
+ var LIFT_DENSITY_POINTS = 160;
225
+ var LIFT_INTEGRATION_STEPS = 400;
226
+ function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
227
+ const lower = quantile(sortedRelativeLifts, 2e-3);
228
+ const upper = quantile(sortedRelativeLifts, 0.998);
229
+ const span = upper - lower;
230
+ if (!Number.isFinite(span) || span <= 0) {
231
+ return [
232
+ { relativeLift: lower, density: 0 },
233
+ { relativeLift: lower, density: 1 },
234
+ { relativeLift: lower, density: 0 }
235
+ ];
236
+ }
237
+ const totalA = alphaA + betaA;
238
+ const meanA = alphaA / totalA;
239
+ const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
240
+ const aLow = Math.max(1e-9, meanA - 10 * sdA);
241
+ const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
242
+ const lifts = Array.from(
243
+ { length: LIFT_DENSITY_POINTS },
244
+ (_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
245
+ );
246
+ const densities = lifts.map((relativeLift) => {
247
+ const ratio = 1 + relativeLift;
248
+ if (ratio <= 0) return 0;
249
+ const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
250
+ if (aHigh <= aLow) return 0;
251
+ const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
252
+ let total = 0;
253
+ for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
254
+ const a = aLow + step * index;
255
+ const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
256
+ total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
257
+ }
258
+ return total * step / 3;
259
+ });
260
+ const peak = Math.max(...densities, Number.MIN_VALUE);
261
+ return lifts.map((relativeLift, index) => ({
262
+ relativeLift,
263
+ density: densities[index] / peak
264
+ }));
265
+ }
266
+
154
267
  // src/core/plan.ts
155
268
  var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
156
269
  function requiredSamplePerVariant({
@@ -197,6 +310,10 @@ function shareUrl(inputs) {
197
310
  thresholdPercent: formatPercentParam(inputs.threshold),
198
311
  meaningfulLiftPercent: formatPercentParam(inputs.minEffect)
199
312
  });
313
+ if (inputs.prior && !(inputs.prior[0] === 1 && inputs.prior[1] === 1)) {
314
+ params.set("priorAlpha", String(inputs.prior[0]));
315
+ params.set("priorBeta", String(inputs.prior[1]));
316
+ }
200
317
  return `${WEB_TOOL_URL}?${params.toString()}`;
201
318
  }
202
319
 
@@ -208,13 +325,23 @@ var PRIOR_ALPHA = 1;
208
325
  var PRIOR_BETA = 1;
209
326
  var POSTERIOR_CHART_POINTS = 96;
210
327
  var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
211
- var WAITING_VALUE_SIMULATIONS = 1500;
328
+ var WAITING_VALUE_SIMULATIONS = 6e3;
212
329
  var DECISION_LABELS = {
213
330
  ship_b: "Ship Variant B",
214
331
  keep_a: "Keep Variant A",
215
332
  continue: "Continue test",
216
333
  inconclusive: "Inconclusive"
217
334
  };
335
+ function validatePrior(prior) {
336
+ const [alpha, beta] = prior;
337
+ if (!(Number.isFinite(alpha) && alpha > 0 && Number.isFinite(beta) && beta > 0)) {
338
+ throw new RangeError(`Prior parameters must be finite and greater than 0, got Beta(${alpha}, ${beta}).`);
339
+ }
340
+ return [alpha, beta];
341
+ }
342
+ function isDefaultPrior([alpha, beta]) {
343
+ return alpha === PRIOR_ALPHA && beta === PRIOR_BETA;
344
+ }
218
345
  function deriveSeed(...parts) {
219
346
  let hash = 2166136261;
220
347
  for (const part of parts) {
@@ -269,8 +396,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
269
396
  });
270
397
  }
271
398
  function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
272
- const random = makeDeterministicRandom(alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19);
399
+ const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
273
400
  return WAITING_SCENARIOS.map((extraPerVariant) => {
401
+ const random = makeDeterministicRandom(seed);
274
402
  let regretSum = 0;
275
403
  for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
276
404
  const trueRateA = sampleBeta(alphaA, betaA, random);
@@ -296,12 +424,14 @@ function analyze({
296
424
  minEffect = DEFAULT_MIN_EFFECT,
297
425
  samples = DEFAULT_SAMPLES,
298
426
  seed = null,
299
- includeChart = false
427
+ includeChart = false,
428
+ prior = [PRIOR_ALPHA, PRIOR_BETA]
300
429
  }) {
301
- const alphaA = PRIOR_ALPHA + conversionsA;
302
- const betaA = PRIOR_BETA + (visitorsA - conversionsA);
303
- const alphaB = PRIOR_ALPHA + conversionsB;
304
- const betaB = PRIOR_BETA + (visitorsB - conversionsB);
430
+ const [priorAlpha, priorBeta] = validatePrior(prior);
431
+ const alphaA = priorAlpha + conversionsA;
432
+ const betaA = priorBeta + (visitorsA - conversionsA);
433
+ const alphaB = priorAlpha + conversionsB;
434
+ const betaB = priorBeta + (visitorsB - conversionsB);
305
435
  const rateA = conversionsA / visitorsA;
306
436
  const rateB = conversionsB / visitorsB;
307
437
  const posteriorMeanA = alphaA / (alphaA + betaA);
@@ -336,7 +466,7 @@ function analyze({
336
466
  conversionsB,
337
467
  threshold,
338
468
  minEffect,
339
- prior: [PRIOR_ALPHA, PRIOR_BETA],
469
+ prior: [priorAlpha, priorBeta],
340
470
  samples,
341
471
  seed: resolvedSeed
342
472
  },
@@ -354,7 +484,9 @@ function analyze({
354
484
  aBeatsB: 1 - pBBeatsA,
355
485
  meaningfulLift: probabilityMeaningfulLift,
356
486
  meaningfulHarm: probabilityMeaningfulHarm,
357
- harmful: share((lift) => lift < 0),
487
+ /* The exact complement of the closed-form win probability. Estimating it
488
+ from the samples left the two halves summing to slightly less than 1. */
489
+ harmful: 1 - pBBeatsA,
358
490
  liftAbove1pct: share((lift) => lift > 0.01),
359
491
  liftAbove5pct: share((lift) => lift > 0.05),
360
492
  harmBeyond5pct: share((lift) => lift < -0.05),
@@ -366,7 +498,9 @@ function analyze({
366
498
  absoluteDiff: posteriorMeanB - posteriorMeanA,
367
499
  absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
368
500
  observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
369
- observedAbsoluteDiff: rateB - rateA
501
+ observedAbsoluteDiff: rateB - rateA,
502
+ relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
503
+ relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
370
504
  },
371
505
  expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
372
506
  variants: {
@@ -380,12 +514,13 @@ function analyze({
380
514
  },
381
515
  waiting: waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow),
382
516
  frequentist: { pValueTwoSided: twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) },
383
- shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect })
517
+ shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] })
384
518
  };
385
519
  if (includeChart) {
386
520
  const sortedA = [...samplesA].sort((left, right) => left - right);
387
521
  const sortedB = [...samplesB].sort((left, right) => left - right);
388
522
  result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
523
+ result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
389
524
  }
390
525
  return result;
391
526
  }
@@ -467,9 +602,11 @@ function project({
467
602
  stopRule = "lift",
468
603
  maxLoss = DEFAULT_MAX_LOSS,
469
604
  simulations = DEFAULT_SIMULATIONS,
470
- seed = null
605
+ seed = null,
606
+ prior = [PRIOR_ALPHA, PRIOR_BETA]
471
607
  }) {
472
- const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
608
+ const [priorAlpha, priorBeta] = validatePrior(prior);
609
+ const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
473
610
  const resolvedSeed = seed ?? deriveSeed(visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, ["lift", "superiority", "loss"].indexOf(stopRule), simulations);
474
611
  const inputs = {
475
612
  visitorsA,
@@ -478,7 +615,7 @@ function project({
478
615
  conversionsB,
479
616
  threshold,
480
617
  minEffect,
481
- prior: [PRIOR_ALPHA, PRIOR_BETA],
618
+ prior: [priorAlpha, priorBeta],
482
619
  dailyVisitors,
483
620
  horizonDays,
484
621
  stopRule,
@@ -486,7 +623,7 @@ function project({
486
623
  simulations,
487
624
  seed: resolvedSeed
488
625
  };
489
- const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
626
+ const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
490
627
  const diagnostic = stopRule === "lift" && liftNearThreshold(current, minEffect) ? "lift_near_threshold" : null;
491
628
  const decisionNow = decisionNowFrom(current, stopRule, threshold, maxLoss);
492
629
  if (decisionNow) {
@@ -503,10 +640,10 @@ function project({
503
640
  shareUrl: link
504
641
  };
505
642
  }
506
- const alphaA0 = PRIOR_ALPHA + conversionsA;
507
- const betaA0 = PRIOR_BETA + (visitorsA - conversionsA);
508
- const alphaB0 = PRIOR_ALPHA + conversionsB;
509
- const betaB0 = PRIOR_BETA + (visitorsB - conversionsB);
643
+ const alphaA0 = priorAlpha + conversionsA;
644
+ const betaA0 = priorBeta + (visitorsA - conversionsA);
645
+ const alphaB0 = priorAlpha + conversionsB;
646
+ const betaB0 = priorBeta + (visitorsB - conversionsB);
510
647
  const random = makeDeterministicRandom(resolvedSeed);
511
648
  const decidedByDay = new Array(horizonDays + 1).fill(0);
512
649
  const decisionDays = [];
@@ -562,6 +699,30 @@ function project({
562
699
  shareUrl: link
563
700
  };
564
701
  }
702
+
703
+ // src/core/loss.ts
704
+ function integratedExpectedLoss(alphaA, betaA, alphaB, betaB) {
705
+ const GRID = 600;
706
+ const dx = 1 / GRID;
707
+ let cdfA = 0;
708
+ let cdfB = 0;
709
+ let weightedSumA = 0;
710
+ let weightedSumB = 0;
711
+ let shipB = 0;
712
+ let shipA = 0;
713
+ for (let i = 0; i <= GRID; i += 1) {
714
+ const x = clamp(i * dx, 1e-6, 1 - 1e-6);
715
+ const fA = betaPdf(x, alphaA, betaA);
716
+ const fB = betaPdf(x, alphaB, betaB);
717
+ shipB += fA * (x * cdfB - weightedSumB) * dx;
718
+ shipA += fB * (x * cdfA - weightedSumA) * dx;
719
+ cdfA += fA * dx;
720
+ weightedSumA += x * fA * dx;
721
+ cdfB += fB * dx;
722
+ weightedSumB += x * fB * dx;
723
+ }
724
+ return { shipA: Math.max(shipA, 0), shipB: Math.max(shipB, 0) };
725
+ }
565
726
  export {
566
727
  DECISION_LABELS,
567
728
  DEFAULT_HORIZON_DAYS,
@@ -570,11 +731,21 @@ export {
570
731
  DEFAULT_SAMPLES,
571
732
  DEFAULT_SIMULATIONS,
572
733
  DEFAULT_THRESHOLD,
734
+ PRIOR_ALPHA,
735
+ PRIOR_BETA,
573
736
  WEB_TOOL_URL,
574
737
  analyze,
738
+ betaPdf,
739
+ integratedExpectedLoss,
740
+ isDefaultPrior,
741
+ liftDensity,
742
+ logBeta,
743
+ logGamma,
575
744
  plan,
576
745
  probabilityBBeatsA,
577
746
  project,
747
+ regularizedIncompleteBeta,
578
748
  requiredSamplePerVariant,
579
- shareUrl
749
+ shareUrl,
750
+ validatePrior
580
751
  };
package/package.json CHANGED
@@ -1,8 +1,15 @@
1
1
  {
2
2
  "name": "bayes-ab",
3
- "version": "0.1.0",
3
+ "version": "0.3.0",
4
4
  "description": "Bayesian A/B test analysis on the command line, for humans and agents",
5
- "keywords": ["ab-testing", "bayesian", "statistics", "cli", "experimentation", "conversion-rate"],
5
+ "keywords": [
6
+ "ab-testing",
7
+ "bayesian",
8
+ "statistics",
9
+ "cli",
10
+ "experimentation",
11
+ "conversion-rate"
12
+ ],
6
13
  "homepage": "https://www.joebrundage.com/tools/bayesian-ab-test",
7
14
  "repository": {
8
15
  "type": "git",
@@ -23,7 +30,11 @@
23
30
  "import": "./dist/index.js"
24
31
  }
25
32
  },
26
- "files": ["dist", "README.md", "LICENSE"],
33
+ "files": [
34
+ "dist",
35
+ "README.md",
36
+ "LICENSE"
37
+ ],
27
38
  "engines": {
28
39
  "node": ">=18.11"
29
40
  },