bayes-ab 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,580 @@
1
+ // src/core/math.ts
2
+ var LANCZOS_COEFFICIENTS = [
3
+ 676.5203681218851,
4
+ -1259.1392167224028,
5
+ 771.3234287776531,
6
+ -176.6150291621406,
7
+ 12.507343278686905,
8
+ -0.13857109526572012,
9
+ 9984369578019572e-21,
10
+ 15056327351493116e-23
11
+ ];
12
+ function clamp(value, min, max) {
13
+ if (!Number.isFinite(value)) return min;
14
+ if (value < min) return min;
15
+ if (value > max) return max;
16
+ return value;
17
+ }
18
+ function logGamma(z) {
19
+ if (z < 0.5) {
20
+ return Math.log(Math.PI) - Math.log(Math.sin(Math.PI * z)) - logGamma(1 - z);
21
+ }
22
+ const shifted = z - 1;
23
+ let x = 0.9999999999998099;
24
+ for (let index = 0; index < LANCZOS_COEFFICIENTS.length; index += 1) {
25
+ x += LANCZOS_COEFFICIENTS[index] / (shifted + index + 1);
26
+ }
27
+ const t = shifted + LANCZOS_COEFFICIENTS.length - 0.5;
28
+ return 0.9189385332046727 + (shifted + 0.5) * Math.log(t) - t + Math.log(x);
29
+ }
30
+ function logBeta(a, b) {
31
+ return logGamma(a) + logGamma(b) - logGamma(a + b);
32
+ }
33
+ function betaPdf(x, alpha, beta) {
34
+ const boundedX = clamp(x, 1e-9, 1 - 1e-9);
35
+ return Math.exp(
36
+ (alpha - 1) * Math.log(boundedX) + (beta - 1) * Math.log(1 - boundedX) - logBeta(alpha, beta)
37
+ );
38
+ }
39
+ function makeDeterministicRandom(seed) {
40
+ let state = seed >>> 0;
41
+ return () => {
42
+ state = 1664525 * state + 1013904223 >>> 0;
43
+ return state / 4294967296;
44
+ };
45
+ }
46
+ function sampleStandardNormal(random) {
47
+ const u1 = clamp(random(), 1e-12, 1 - 1e-12);
48
+ const u2 = random();
49
+ return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
50
+ }
51
+ function sampleGamma(shape, random) {
52
+ if (shape < 1) {
53
+ const u = clamp(random(), 1e-12, 1 - 1e-12);
54
+ return sampleGamma(shape + 1, random) * u ** (1 / shape);
55
+ }
56
+ const d = shape - 1 / 3;
57
+ const c = 1 / Math.sqrt(9 * d);
58
+ while (true) {
59
+ const x = sampleStandardNormal(random);
60
+ const v = (1 + c * x) ** 3;
61
+ if (v <= 0) continue;
62
+ const u = random();
63
+ if (u < 1 - 0.0331 * x ** 4) return d * v;
64
+ if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v;
65
+ }
66
+ }
67
+ function sampleBeta(alpha, beta, random) {
68
+ const x = sampleGamma(alpha, random);
69
+ const y = sampleGamma(beta, random);
70
+ return x / (x + y);
71
+ }
72
+ function sampleBinomialApprox(trials, probability, random) {
73
+ const boundedTrials = Math.max(0, Math.round(trials));
74
+ const boundedProbability = clamp(probability, 0, 1);
75
+ if (boundedTrials === 0 || boundedProbability === 0) return 0;
76
+ if (boundedProbability === 1) return boundedTrials;
77
+ if (boundedTrials <= 50) {
78
+ let successes = 0;
79
+ for (let index = 0; index < boundedTrials; index += 1) {
80
+ if (random() < boundedProbability) successes += 1;
81
+ }
82
+ return successes;
83
+ }
84
+ const mean2 = boundedTrials * boundedProbability;
85
+ const standardDeviation = Math.sqrt(boundedTrials * boundedProbability * (1 - boundedProbability));
86
+ return clamp(Math.round(mean2 + sampleStandardNormal(random) * standardDeviation), 0, boundedTrials);
87
+ }
88
+ function quantile(sortedValues, probability) {
89
+ const boundedProbability = clamp(probability, 0, 1);
90
+ const position = (sortedValues.length - 1) * boundedProbability;
91
+ const lowerIndex = Math.floor(position);
92
+ const upperIndex = Math.ceil(position);
93
+ const lower = sortedValues[lowerIndex];
94
+ if (lowerIndex === upperIndex) return lower;
95
+ const upper = sortedValues[upperIndex];
96
+ const weight = position - lowerIndex;
97
+ return lower * (1 - weight) + upper * weight;
98
+ }
99
+ function mean(values) {
100
+ let sum = 0;
101
+ for (const value of values) sum += value;
102
+ return sum / values.length;
103
+ }
104
+ function probabilityRightBeatsLeftDirect(alphaLeft, betaLeft, alphaRight, betaRight) {
105
+ const base = logBeta(alphaLeft, betaLeft);
106
+ let maxLog = Number.NEGATIVE_INFINITY;
107
+ let scaledSum = 0;
108
+ for (let i = 0; i <= alphaRight - 1; i += 1) {
109
+ const current = logBeta(alphaLeft + i, betaLeft + betaRight) - Math.log(betaRight + i) - logBeta(1 + i, betaRight) - base;
110
+ if (current <= maxLog) {
111
+ scaledSum += Math.exp(current - maxLog);
112
+ continue;
113
+ }
114
+ scaledSum = scaledSum * Math.exp(maxLog - current) + 1;
115
+ maxLog = current;
116
+ }
117
+ return Math.exp(maxLog) * scaledSum;
118
+ }
119
+ function probabilityBBeatsA(alphaA, betaA, alphaB, betaB) {
120
+ if (alphaB <= alphaA) {
121
+ return probabilityRightBeatsLeftDirect(alphaA, betaA, alphaB, betaB);
122
+ }
123
+ return 1 - probabilityRightBeatsLeftDirect(alphaB, betaB, alphaA, betaA);
124
+ }
125
+ function inverseNormalCdf(p) {
126
+ const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
127
+ const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
128
+ const c = [-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968, 2.938163982698783];
129
+ const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
130
+ const probability = clamp(p, 1e-9, 1 - 1e-9);
131
+ const pLow = 0.02425;
132
+ const pHigh = 1 - pLow;
133
+ const tail = (q2) => (((((c[0] * q2 + c[1]) * q2 + c[2]) * q2 + c[3]) * q2 + c[4]) * q2 + c[5]) / ((((d[0] * q2 + d[1]) * q2 + d[2]) * q2 + d[3]) * q2 + 1);
134
+ if (probability < pLow) return tail(Math.sqrt(-2 * Math.log(probability)));
135
+ if (probability > pHigh) return -tail(Math.sqrt(-2 * Math.log(1 - probability)));
136
+ const q = probability - 0.5;
137
+ const r = q * q;
138
+ return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
139
+ }
140
+ function erf(x) {
141
+ const sign = x < 0 ? -1 : 1;
142
+ const absoluteX = Math.abs(x);
143
+ const t = 1 / (1 + 0.3275911 * absoluteX);
144
+ const y = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-absoluteX * absoluteX);
145
+ return sign * y;
146
+ }
147
+ function normalCdf(x) {
148
+ return 0.5 * (1 + erf(x / Math.SQRT2));
149
+ }
150
+ function normalPdf(x) {
151
+ return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
152
+ }
153
+
154
+ // src/core/plan.ts
155
+ var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
156
+ function requiredSamplePerVariant({
157
+ baseline,
158
+ minEffect,
159
+ alpha = 0.05,
160
+ power = 0.8
161
+ }) {
162
+ const boundedBaseline = clamp(baseline, 1e-6, 1 - 1e-6);
163
+ const absoluteDelta = Math.max(boundedBaseline * minEffect, 1e-6);
164
+ const zAlpha = inverseNormalCdf(1 - alpha / 2);
165
+ const zPower = inverseNormalCdf(power);
166
+ const standardDeviation = Math.sqrt(2 * boundedBaseline * (1 - boundedBaseline));
167
+ return Math.ceil(((zAlpha + zPower) * standardDeviation / absoluteDelta) ** 2);
168
+ }
169
+ function plan({ baseline, minEffect, power = 0.8, alpha = 0.05, dailyVisitors = null }) {
170
+ const daysFor = (perVariant2) => dailyVisitors && dailyVisitors > 0 ? Math.ceil(perVariant2 / dailyVisitors) : null;
171
+ const perVariant = requiredSamplePerVariant({ baseline, minEffect, alpha, power });
172
+ const effects = SENSITIVITY_EFFECTS.includes(minEffect) ? SENSITIVITY_EFFECTS : [...SENSITIVITY_EFFECTS, minEffect].sort((left, right) => left - right);
173
+ return {
174
+ inputs: { baseline, minEffect, power, alpha, dailyVisitors },
175
+ targetRates: { up: baseline * (1 + minEffect), down: baseline * (1 - minEffect) },
176
+ perVariant,
177
+ total: perVariant * 2,
178
+ days: daysFor(perVariant),
179
+ sensitivity: effects.map((effect) => {
180
+ const rowPerVariant = requiredSamplePerVariant({ baseline, minEffect: effect, alpha, power });
181
+ return { minEffect: effect, perVariant: rowPerVariant, days: daysFor(rowPerVariant) };
182
+ })
183
+ };
184
+ }
185
+
186
+ // src/core/share.ts
187
+ var WEB_TOOL_URL = "https://www.joebrundage.com/tools/bayesian-ab-test";
188
+ function formatPercentParam(fraction) {
189
+ return String(Number((fraction * 100).toPrecision(10)));
190
+ }
191
+ function shareUrl(inputs) {
192
+ const params = new URLSearchParams({
193
+ visitorsA: String(inputs.visitorsA),
194
+ conversionsA: String(inputs.conversionsA),
195
+ visitorsB: String(inputs.visitorsB),
196
+ conversionsB: String(inputs.conversionsB),
197
+ thresholdPercent: formatPercentParam(inputs.threshold),
198
+ meaningfulLiftPercent: formatPercentParam(inputs.minEffect)
199
+ });
200
+ return `${WEB_TOOL_URL}?${params.toString()}`;
201
+ }
202
+
203
+ // src/core/analyze.ts
204
+ var DEFAULT_THRESHOLD = 0.95;
205
+ var DEFAULT_MIN_EFFECT = 0.1;
206
+ var DEFAULT_SAMPLES = 12e3;
207
+ var PRIOR_ALPHA = 1;
208
+ var PRIOR_BETA = 1;
209
+ var POSTERIOR_CHART_POINTS = 96;
210
+ var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
211
+ var WAITING_VALUE_SIMULATIONS = 1500;
212
+ var DECISION_LABELS = {
213
+ ship_b: "Ship Variant B",
214
+ keep_a: "Keep Variant A",
215
+ continue: "Continue test",
216
+ inconclusive: "Inconclusive"
217
+ };
218
+ function deriveSeed(...parts) {
219
+ let hash = 2166136261;
220
+ for (const part of parts) {
221
+ const text = String(part);
222
+ for (let index = 0; index < text.length; index += 1) {
223
+ hash ^= text.charCodeAt(index);
224
+ hash = Math.imul(hash, 16777619) >>> 0;
225
+ }
226
+ hash ^= 2654435769;
227
+ hash = Math.imul(hash, 16777619) >>> 0;
228
+ }
229
+ return hash >>> 0;
230
+ }
231
+ function twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) {
232
+ const pA = conversionsA / visitorsA;
233
+ const pB = conversionsB / visitorsB;
234
+ const pooled = (conversionsA + conversionsB) / (visitorsA + visitorsB);
235
+ const standardError = Math.sqrt(pooled * (1 - pooled) * (1 / visitorsA + 1 / visitorsB));
236
+ if (standardError === 0) return 1;
237
+ const z = (pB - pA) / standardError;
238
+ return clamp(2 * (1 - normalCdf(Math.abs(z))), 0, 1);
239
+ }
240
+ function evidenceStrength(current, required) {
241
+ const ratio = current / Math.max(required, 1);
242
+ if (ratio < 0.5) return "low";
243
+ if (ratio < 1) return "moderate";
244
+ return "strong";
245
+ }
246
+ function maturity(progress) {
247
+ if (progress < 0.35) return "early";
248
+ if (progress < 0.85) return "building";
249
+ return "mature";
250
+ }
251
+ function decisionStatus({
252
+ probabilityMeaningfulLift,
253
+ probabilityMeaningfulHarm,
254
+ threshold,
255
+ evidence
256
+ }) {
257
+ if (probabilityMeaningfulLift >= threshold) return "ship_b";
258
+ if (probabilityMeaningfulHarm >= threshold) return "keep_a";
259
+ if (evidence === "low") return "continue";
260
+ return "inconclusive";
261
+ }
262
+ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
263
+ const minRate = Math.max(1e-4, Math.min(quantile(sortedA, 1e-3), quantile(sortedB, 1e-3)) * 0.9);
264
+ const maxRate = Math.min(0.9999, Math.max(quantile(sortedA, 0.999), quantile(sortedB, 0.999)) * 1.1);
265
+ const span = maxRate - minRate;
266
+ return Array.from({ length: POSTERIOR_CHART_POINTS }, (_, index) => {
267
+ const rate = minRate + span * index / (POSTERIOR_CHART_POINTS - 1);
268
+ return { rate, densityA: betaPdf(rate, alphaA, betaA), densityB: betaPdf(rate, alphaB, betaB) };
269
+ });
270
+ }
271
+ function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
272
+ const random = makeDeterministicRandom(alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19);
273
+ return WAITING_SCENARIOS.map((extraPerVariant) => {
274
+ let regretSum = 0;
275
+ for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
276
+ const trueRateA = sampleBeta(alphaA, betaA, random);
277
+ const trueRateB = sampleBeta(alphaB, betaB, random);
278
+ const extraA = sampleBinomialApprox(extraPerVariant, trueRateA, random);
279
+ const extraB = sampleBinomialApprox(extraPerVariant, trueRateB, random);
280
+ const futureMeanA = (alphaA + extraA) / (alphaA + betaA + extraPerVariant);
281
+ const futureMeanB = (alphaB + extraB) / (alphaB + betaB + extraPerVariant);
282
+ const regretIfChooseA = Math.max(trueRateB - trueRateA, 0);
283
+ const regretIfChooseB = Math.max(trueRateA - trueRateB, 0);
284
+ regretSum += futureMeanB >= futureMeanA ? regretIfChooseB : regretIfChooseA;
285
+ }
286
+ const expectedRegret = regretSum / WAITING_VALUE_SIMULATIONS;
287
+ return { extraPerVariant, expectedRegret, valueOfWaiting: Math.max(0, expectedRegretNow - expectedRegret) };
288
+ });
289
+ }
290
+ function analyze({
291
+ visitorsA,
292
+ conversionsA,
293
+ visitorsB,
294
+ conversionsB,
295
+ threshold = DEFAULT_THRESHOLD,
296
+ minEffect = DEFAULT_MIN_EFFECT,
297
+ samples = DEFAULT_SAMPLES,
298
+ seed = null,
299
+ includeChart = false
300
+ }) {
301
+ const alphaA = PRIOR_ALPHA + conversionsA;
302
+ const betaA = PRIOR_BETA + (visitorsA - conversionsA);
303
+ const alphaB = PRIOR_ALPHA + conversionsB;
304
+ const betaB = PRIOR_BETA + (visitorsB - conversionsB);
305
+ const rateA = conversionsA / visitorsA;
306
+ const rateB = conversionsB / visitorsB;
307
+ const posteriorMeanA = alphaA / (alphaA + betaA);
308
+ const posteriorMeanB = alphaB / (alphaB + betaB);
309
+ const pBBeatsA = clamp(probabilityBBeatsA(alphaA, betaA, alphaB, betaB), 0, 1);
310
+ const resolvedSeed = seed ?? alphaA + betaA * 3 + alphaB * 5 + betaB * 7;
311
+ const random = makeDeterministicRandom(resolvedSeed);
312
+ const samplesA = Array.from({ length: samples }, () => sampleBeta(alphaA, betaA, random));
313
+ const samplesB = Array.from({ length: samples }, () => sampleBeta(alphaB, betaB, random));
314
+ const relativeLifts = samplesA.map((sampleA, index) => samplesB[index] / sampleA - 1);
315
+ const absoluteDiffs = samplesA.map((sampleA, index) => samplesB[index] - sampleA);
316
+ const sortedRelativeLifts = [...relativeLifts].sort((left, right) => left - right);
317
+ const sortedAbsoluteDiffs = [...absoluteDiffs].sort((left, right) => left - right);
318
+ const share = (predicate) => mean(relativeLifts.map((lift) => predicate(lift) ? 1 : 0));
319
+ const expectedLossShipA = mean(absoluteDiffs.map((difference) => Math.max(difference, 0)));
320
+ const expectedLossShipB = mean(absoluteDiffs.map((difference) => Math.max(-difference, 0)));
321
+ const bayesActionNow = expectedLossShipB <= expectedLossShipA ? "B" : "A";
322
+ const expectedRegretNow = bayesActionNow === "B" ? expectedLossShipB : expectedLossShipA;
323
+ const baselineRate = (posteriorMeanA + posteriorMeanB) / 2;
324
+ const smallerArm = Math.min(visitorsA, visitorsB);
325
+ const requiredPerVariant = requiredSamplePerVariant({ baseline: baselineRate, minEffect });
326
+ const evidence = evidenceStrength(smallerArm, requiredPerVariant);
327
+ const maturityProgress = clamp(smallerArm / Math.max(requiredPerVariant, 1), 0, 1);
328
+ const probabilityMeaningfulLift = share((lift) => lift > minEffect);
329
+ const probabilityMeaningfulHarm = share((lift) => lift < -minEffect);
330
+ const status = decisionStatus({ probabilityMeaningfulLift, probabilityMeaningfulHarm, threshold, evidence });
331
+ const result = {
332
+ inputs: {
333
+ visitorsA,
334
+ conversionsA,
335
+ visitorsB,
336
+ conversionsB,
337
+ threshold,
338
+ minEffect,
339
+ prior: [PRIOR_ALPHA, PRIOR_BETA],
340
+ samples,
341
+ seed: resolvedSeed
342
+ },
343
+ decision: {
344
+ status,
345
+ label: DECISION_LABELS[status],
346
+ evidence,
347
+ maturity: maturity(maturityProgress),
348
+ maturityProgress,
349
+ bayesActionNow,
350
+ expectedRegretNow
351
+ },
352
+ probability: {
353
+ bBeatsA: pBBeatsA,
354
+ aBeatsB: 1 - pBBeatsA,
355
+ meaningfulLift: probabilityMeaningfulLift,
356
+ meaningfulHarm: probabilityMeaningfulHarm,
357
+ harmful: share((lift) => lift < 0),
358
+ liftAbove1pct: share((lift) => lift > 0.01),
359
+ liftAbove5pct: share((lift) => lift > 0.05),
360
+ harmBeyond5pct: share((lift) => lift < -0.05),
361
+ harmBeyond10pct: share((lift) => lift < -0.1)
362
+ },
363
+ effect: {
364
+ relativeLift: posteriorMeanB / posteriorMeanA - 1,
365
+ relativeLiftCI95: [quantile(sortedRelativeLifts, 0.025), quantile(sortedRelativeLifts, 0.975)],
366
+ absoluteDiff: posteriorMeanB - posteriorMeanA,
367
+ absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
368
+ observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
369
+ observedAbsoluteDiff: rateB - rateA
370
+ },
371
+ expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
372
+ variants: {
373
+ A: { visitors: visitorsA, conversions: conversionsA, rate: rateA, posteriorMean: posteriorMeanA, posterior: [alphaA, betaA] },
374
+ B: { visitors: visitorsB, conversions: conversionsB, rate: rateB, posteriorMean: posteriorMeanB, posterior: [alphaB, betaB] }
375
+ },
376
+ sampleSize: {
377
+ requiredPerVariant,
378
+ additionalPerVariant: Math.max(0, requiredPerVariant - smallerArm),
379
+ detectableEffectNow: (inverseNormalCdf(1 - 0.05 / 2) + inverseNormalCdf(0.8)) * Math.sqrt(2 * baselineRate * (1 - baselineRate) / smallerArm) / baselineRate
380
+ },
381
+ waiting: waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow),
382
+ frequentist: { pValueTwoSided: twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) },
383
+ shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect })
384
+ };
385
+ if (includeChart) {
386
+ const sortedA = [...samplesA].sort((left, right) => left - right);
387
+ const sortedB = [...samplesB].sort((left, right) => left - right);
388
+ result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
389
+ }
390
+ return result;
391
+ }
392
+
393
+ // src/core/project.ts
394
+ var DEFAULT_HORIZON_DAYS = 28;
395
+ var DEFAULT_SIMULATIONS = 2e3;
396
+ var DEFAULT_MAX_LOSS = 1e-4;
397
+ function moments({ alpha, beta }) {
398
+ const total = alpha + beta;
399
+ return { mean: alpha / total, variance: alpha * beta / (total * total * (total + 1)) };
400
+ }
401
+ function probabilityBExceedsScaledA(a, b, k) {
402
+ const mA = moments(a);
403
+ const mB = moments(b);
404
+ const mean2 = mB.mean - k * mA.mean;
405
+ const sd = Math.sqrt(mB.variance + k * k * mA.variance);
406
+ if (sd === 0) return mean2 > 0 ? 1 : 0;
407
+ return normalCdf(mean2 / sd);
408
+ }
409
+ function expectedLosses(a, b) {
410
+ const mA = moments(a);
411
+ const mB = moments(b);
412
+ const mean2 = mB.mean - mA.mean;
413
+ const sd = Math.sqrt(mA.variance + mB.variance);
414
+ if (sd === 0) return { shipA: Math.max(mean2, 0), shipB: Math.max(-mean2, 0) };
415
+ const z = mean2 / sd;
416
+ return {
417
+ shipA: mean2 * normalCdf(z) + sd * normalPdf(z),
418
+ shipB: -mean2 * normalCdf(-z) + sd * normalPdf(z)
419
+ };
420
+ }
421
+ function evaluateRule(a, b, rule, threshold, minEffect, maxLoss) {
422
+ if (rule === "loss") {
423
+ const loss = expectedLosses(a, b);
424
+ if (Math.min(loss.shipA, loss.shipB) > maxLoss) return null;
425
+ return loss.shipB <= loss.shipA ? "ship_b" : "keep_a";
426
+ }
427
+ const effect = rule === "lift" ? minEffect : 0;
428
+ if (probabilityBExceedsScaledA(a, b, 1 + effect) >= threshold) return "ship_b";
429
+ if (1 - probabilityBExceedsScaledA(a, b, 1 - effect) >= threshold) return "keep_a";
430
+ return null;
431
+ }
432
+ function decisionNowFrom(current, rule, threshold, maxLoss) {
433
+ if (rule === "lift") {
434
+ if (current.decision.status === "ship_b" || current.decision.status === "keep_a") return current.decision.status;
435
+ return null;
436
+ }
437
+ if (rule === "superiority") {
438
+ if (current.probability.bBeatsA >= threshold) return "ship_b";
439
+ if (current.probability.aBeatsB >= threshold) return "keep_a";
440
+ return null;
441
+ }
442
+ const { shipA, shipB } = current.expectedLoss;
443
+ if (Math.min(shipA, shipB) > maxLoss) return null;
444
+ return shipB <= shipA ? "ship_b" : "keep_a";
445
+ }
446
+ function checkpoints(horizonDays) {
447
+ const days = [];
448
+ for (let day = 7; day < horizonDays; day += 7) days.push(day);
449
+ days.push(horizonDays);
450
+ return days;
451
+ }
452
+ function liftNearThreshold(current, minEffect) {
453
+ const lift = current.effect.relativeLift;
454
+ const [lower, upper] = current.effect.relativeLiftCI95;
455
+ const distance = Math.min(Math.abs(lift - minEffect), Math.abs(lift + minEffect));
456
+ return distance < (upper - lower) / 8;
457
+ }
458
+ function project({
459
+ visitorsA,
460
+ conversionsA,
461
+ visitorsB,
462
+ conversionsB,
463
+ dailyVisitors,
464
+ horizonDays = DEFAULT_HORIZON_DAYS,
465
+ threshold = DEFAULT_THRESHOLD,
466
+ minEffect = DEFAULT_MIN_EFFECT,
467
+ stopRule = "lift",
468
+ maxLoss = DEFAULT_MAX_LOSS,
469
+ simulations = DEFAULT_SIMULATIONS,
470
+ seed = null
471
+ }) {
472
+ const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
473
+ const resolvedSeed = seed ?? deriveSeed(visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, ["lift", "superiority", "loss"].indexOf(stopRule), simulations);
474
+ const inputs = {
475
+ visitorsA,
476
+ conversionsA,
477
+ visitorsB,
478
+ conversionsB,
479
+ threshold,
480
+ minEffect,
481
+ prior: [PRIOR_ALPHA, PRIOR_BETA],
482
+ dailyVisitors,
483
+ horizonDays,
484
+ stopRule,
485
+ maxLoss: stopRule === "loss" ? maxLoss : null,
486
+ simulations,
487
+ seed: resolvedSeed
488
+ };
489
+ const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
490
+ const diagnostic = stopRule === "lift" && liftNearThreshold(current, minEffect) ? "lift_near_threshold" : null;
491
+ const decisionNow = decisionNowFrom(current, stopRule, threshold, maxLoss);
492
+ if (decisionNow) {
493
+ return {
494
+ inputs,
495
+ alreadyDecided: true,
496
+ decisionNow,
497
+ decisionWithinHorizon: 1,
498
+ outcomes: { shipB: decisionNow === "ship_b" ? 1 : 0, keepA: decisionNow === "keep_a" ? 1 : 0, undecided: 0 },
499
+ daysToDecision: { p25: 0, p50: 0, p75: 0, p90: 0 },
500
+ cumulative: checkpoints(horizonDays).map((day) => ({ day, p: 1 })),
501
+ diagnostic,
502
+ approximation: "normal",
503
+ shareUrl: link
504
+ };
505
+ }
506
+ const alphaA0 = PRIOR_ALPHA + conversionsA;
507
+ const betaA0 = PRIOR_BETA + (visitorsA - conversionsA);
508
+ const alphaB0 = PRIOR_ALPHA + conversionsB;
509
+ const betaB0 = PRIOR_BETA + (visitorsB - conversionsB);
510
+ const random = makeDeterministicRandom(resolvedSeed);
511
+ const decidedByDay = new Array(horizonDays + 1).fill(0);
512
+ const decisionDays = [];
513
+ let shipB = 0;
514
+ let keepA = 0;
515
+ for (let simulation = 0; simulation < simulations; simulation += 1) {
516
+ const trueRateA = sampleBeta(alphaA0, betaA0, random);
517
+ const trueRateB = sampleBeta(alphaB0, betaB0, random);
518
+ const a = { alpha: alphaA0, beta: betaA0 };
519
+ const b = { alpha: alphaB0, beta: betaB0 };
520
+ for (let day = 1; day <= horizonDays; day += 1) {
521
+ const newA = sampleBinomialApprox(dailyVisitors, trueRateA, random);
522
+ const newB = sampleBinomialApprox(dailyVisitors, trueRateB, random);
523
+ a.alpha += newA;
524
+ a.beta += dailyVisitors - newA;
525
+ b.alpha += newB;
526
+ b.beta += dailyVisitors - newB;
527
+ const decision = evaluateRule(a, b, stopRule, threshold, minEffect, maxLoss);
528
+ if (decision) {
529
+ if (decision === "ship_b") shipB += 1;
530
+ else keepA += 1;
531
+ decidedByDay[day] = decidedByDay[day] + 1;
532
+ decisionDays.push(day);
533
+ break;
534
+ }
535
+ }
536
+ }
537
+ let running = 0;
538
+ const cumulativeByDay = decidedByDay.map((count) => {
539
+ running += count;
540
+ return running / simulations;
541
+ });
542
+ const sortedDays = [...decisionDays].sort((left, right) => left - right);
543
+ return {
544
+ inputs,
545
+ alreadyDecided: false,
546
+ decisionNow: null,
547
+ decisionWithinHorizon: decisionDays.length / simulations,
548
+ outcomes: {
549
+ shipB: shipB / simulations,
550
+ keepA: keepA / simulations,
551
+ undecided: (simulations - decisionDays.length) / simulations
552
+ },
553
+ daysToDecision: sortedDays.length === 0 ? null : {
554
+ p25: quantile(sortedDays, 0.25),
555
+ p50: quantile(sortedDays, 0.5),
556
+ p75: quantile(sortedDays, 0.75),
557
+ p90: quantile(sortedDays, 0.9)
558
+ },
559
+ cumulative: checkpoints(horizonDays).map((day) => ({ day, p: cumulativeByDay[day] })),
560
+ diagnostic,
561
+ approximation: "normal",
562
+ shareUrl: link
563
+ };
564
+ }
565
+ export {
566
+ DECISION_LABELS,
567
+ DEFAULT_HORIZON_DAYS,
568
+ DEFAULT_MAX_LOSS,
569
+ DEFAULT_MIN_EFFECT,
570
+ DEFAULT_SAMPLES,
571
+ DEFAULT_SIMULATIONS,
572
+ DEFAULT_THRESHOLD,
573
+ WEB_TOOL_URL,
574
+ analyze,
575
+ plan,
576
+ probabilityBBeatsA,
577
+ project,
578
+ requiredSamplePerVariant,
579
+ shareUrl
580
+ };
package/package.json ADDED
@@ -0,0 +1,42 @@
1
+ {
2
+ "name": "bayes-ab",
3
+ "version": "0.1.0",
4
+ "description": "Bayesian A/B test analysis on the command line, for humans and agents",
5
+ "keywords": ["ab-testing", "bayesian", "statistics", "cli", "experimentation", "conversion-rate"],
6
+ "homepage": "https://www.joebrundage.com/tools/bayesian-ab-test",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/brundagejoe/bayes-ab.git"
10
+ },
11
+ "bugs": "https://github.com/brundagejoe/bayes-ab/issues",
12
+ "license": "MIT",
13
+ "author": "Joe Brundage",
14
+ "type": "module",
15
+ "bin": {
16
+ "bayes-ab": "dist/cli.js"
17
+ },
18
+ "main": "dist/index.js",
19
+ "types": "dist/index.d.ts",
20
+ "exports": {
21
+ ".": {
22
+ "types": "./dist/index.d.ts",
23
+ "import": "./dist/index.js"
24
+ }
25
+ },
26
+ "files": ["dist", "README.md", "LICENSE"],
27
+ "engines": {
28
+ "node": ">=18.11"
29
+ },
30
+ "scripts": {
31
+ "build": "node scripts/build.mjs",
32
+ "typecheck": "tsc --noEmit",
33
+ "test": "node --test --experimental-strip-types test/*.test.ts",
34
+ "check": "npm run typecheck && npm test && npm run build",
35
+ "prepublishOnly": "npm run check"
36
+ },
37
+ "devDependencies": {
38
+ "@types/node": "^22.10.0",
39
+ "esbuild": "^0.24.0",
40
+ "typescript": "^5.7.0"
41
+ }
42
+ }