bayes-ab 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -8
- package/dist/cli.js +198 -29
- package/dist/core/analyze.d.ts +10 -1
- package/dist/core/lift.d.ts +15 -0
- package/dist/core/loss.d.ts +13 -0
- package/dist/core/math.d.ts +9 -1
- package/dist/core/project.d.ts +3 -1
- package/dist/core/share.d.ts +1 -0
- package/dist/index.d.ts +5 -2
- package/dist/index.js +191 -20
- package/package.json +14 -3
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ Bayes action now Ship B (expected regret 0.0050 pp, about 0.50 missed conve
|
|
|
23
23
|
P(B beats A) 95.91%
|
|
24
24
|
P(relative lift > +10%) 46.42%
|
|
25
25
|
P(relative lift < -10%) 0.02%
|
|
26
|
-
P(B is harmful at all) 4.
|
|
26
|
+
P(B is harmful at all) 4.09%
|
|
27
27
|
|
|
28
28
|
Relative lift +9.54% 95% CI -1.17% to +21.30%
|
|
29
29
|
Absolute difference +0.53 pp 95% CI -0.07 pp to +1.11 pp
|
|
@@ -98,6 +98,14 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 10000 --rat
|
|
|
98
98
|
# Continue test · P(B beats A) 97.13% · P(lift > +10%) 56.76% · lift +10.92% (CI -0.30% to +23.39%) · loss if ship B 0.0034 pp B = 610/10000 from --rate-b 6.10%
|
|
99
99
|
```
|
|
100
100
|
|
|
101
|
+
Choose a prior. Every analysis defaults to a flat Beta(1,1) on each conversion rate; `--prior-alpha` and `--prior-beta` replace it, the same prior for both variants. Beta(α, β) reads as α conversions in α + β visitors of evidence you already had, so a prior worth 1,000 visitors centred on 5.5% is Beta(55, 945). Fractional values work, e.g. the Jeffreys prior Beta(0.5, 0.5):
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --conversions-b 714 --prior-alpha 55 --prior-beta 945
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
A shared prior pulls both arms toward the same rate, so a strong one shrinks the estimated lift and makes decisions more conservative.
|
|
108
|
+
|
|
101
109
|
### project
|
|
102
110
|
|
|
103
111
|
Takes the current posterior, draws a plausible pair of true conversion rates, simulates each future day of traffic, and re-runs the decision rule after every day. Repeats that thousands of times.
|
|
@@ -173,16 +181,16 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --con
|
|
|
173
181
|
```json
|
|
174
182
|
{
|
|
175
183
|
"tool": "bayes-ab",
|
|
176
|
-
"version": "0.
|
|
184
|
+
"version": "0.3.0",
|
|
177
185
|
"command": "analyze",
|
|
178
|
-
"inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed":
|
|
186
|
+
"inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 116218 },
|
|
179
187
|
"decision": { "status": "continue", "label": "Continue test", "evidence": "low", "maturity": "building", "maturityProgress": 0.462241, "bayesActionNow": "B", "expectedRegretNow": 0.00005 },
|
|
180
|
-
"probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.
|
|
181
|
-
"effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253 },
|
|
188
|
+
"probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040886, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
|
|
189
|
+
"effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253, "relativeLiftMedian": 0.095267, "relativeLiftIQR": [0.056954, 0.134715] },
|
|
182
190
|
"expectedLoss": { "shipA": 0.00528, "shipB": 0.00005 },
|
|
183
191
|
"variants": { "A": { "...": "..." }, "B": { "...": "..." } },
|
|
184
192
|
"sampleSize": { "requiredPerVariant": 25636, "additionalPerVariant": 13786, "detectableEffectNow": 0.147083 },
|
|
185
|
-
"waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.
|
|
193
|
+
"waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.000048, "valueOfWaiting": 0.000002 }, "..." ],
|
|
186
194
|
"frequentist": { "pValueTwoSided": 0.081709 },
|
|
187
195
|
"shareUrl": "https://www.joebrundage.com/tools/bayesian-ab-test?visitorsA=12000&conversionsA=660&visitorsB=11850&conversionsB=714&thresholdPercent=95&meaningfulLiftPercent=10"
|
|
188
196
|
}
|
|
@@ -227,12 +235,30 @@ import { analyze, project, plan } from "bayes-ab"
|
|
|
227
235
|
|
|
228
236
|
const result = analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850, conversionsB: 714 })
|
|
229
237
|
result.probability.bBeatsA // 0.9591138673472563
|
|
238
|
+
|
|
239
|
+
analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850, conversionsB: 714, prior: [55, 945] })
|
|
230
240
|
```
|
|
231
241
|
|
|
242
|
+
`analyze` and `project` take an optional `prior: [alpha, beta]`, default `[1, 1]`. Non-positive values throw a `RangeError`; `validatePrior` and `isDefaultPrior` are exported for forms that want to check first.
|
|
243
|
+
|
|
244
|
+
`analyze({ ..., includeChart: true })` adds `chart` (both posterior densities) and `liftChart` (the exact density of the relative lift, peak scaled to 1), which is what the web tool plots.
|
|
245
|
+
|
|
246
|
+
The primitives behind it are exported too, for anything that needs the same math without a full analysis:
|
|
247
|
+
|
|
248
|
+
| Export | What it does |
|
|
249
|
+
|---|---|
|
|
250
|
+
| `betaPdf(x, alpha, beta)`, `logBeta`, `logGamma` | Beta density and its log-gamma building blocks |
|
|
251
|
+
| `probabilityBBeatsA(alphaA, betaA, alphaB, betaB)` | P(B > A) for two Beta posteriors: exact closed form for whole-number alphas, numerical integration otherwise |
|
|
252
|
+
| `regularizedIncompleteBeta(x, a, b)` | The Beta(a, b) CDF at x |
|
|
253
|
+
| `integratedExpectedLoss(alphaA, betaA, alphaB, betaB)` | Expected loss of shipping each variant by numerical integration, no sampling |
|
|
254
|
+
| `liftDensity(alphaA, betaA, alphaB, betaB, sortedLifts)` | Exact relative-lift density over the range the sorted lifts span |
|
|
255
|
+
|
|
256
|
+
This package is the source of truth for the math on joebrundage.com: the [calculator](https://www.joebrundage.com/tools/bayesian-ab-test) and the [primer](https://www.joebrundage.com/tools/bayesian-ab-testing-primer) both import it. Change the math here, publish, and bump the site's dependency.
|
|
257
|
+
|
|
232
258
|
## Method
|
|
233
259
|
|
|
234
|
-
- Each conversion rate gets a Beta(1,1)
|
|
235
|
-
- P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html).
|
|
260
|
+
- Each conversion rate gets a Beta(α, β) prior, Beta(1,1) unless you choose another. Posteriors are Beta(α + conversions, β + non-conversions).
|
|
261
|
+
- P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html) when α is a whole number. For a fractional α it is integrated numerically (tanh-sinh quadrature over the incomplete beta function), accurate to about 1e-10. P(B is harmful) is its exact complement.
|
|
236
262
|
- Lift intervals, threshold probabilities and expected loss come from 12,000 seeded Monte Carlo draws.
|
|
237
263
|
- The meaningful effect is relative and symmetric: `--min-effect 10` means B must be 10% better than A to count as a win or 10% worse to count as a loss.
|
|
238
264
|
- `project` samples true rates from the exact posterior, then uses a normal approximation to the posterior when re-checking the stopping rule on each simulated day. Day zero is checked exactly.
|
package/dist/cli.js
CHANGED
|
@@ -119,11 +119,80 @@ function probabilityRightBeatsLeftDirect(alphaLeft, betaLeft, alphaRight, betaRi
|
|
|
119
119
|
return Math.exp(maxLog) * scaledSum;
|
|
120
120
|
}
|
|
121
121
|
function probabilityBBeatsA(alphaA, betaA, alphaB, betaB) {
|
|
122
|
+
if (!Number.isInteger(alphaA) || !Number.isInteger(alphaB)) {
|
|
123
|
+
return probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB);
|
|
124
|
+
}
|
|
122
125
|
if (alphaB <= alphaA) {
|
|
123
126
|
return probabilityRightBeatsLeftDirect(alphaA, betaA, alphaB, betaB);
|
|
124
127
|
}
|
|
125
128
|
return 1 - probabilityRightBeatsLeftDirect(alphaB, betaB, alphaA, betaA);
|
|
126
129
|
}
|
|
130
|
+
function betaContinuedFraction(x, a, b) {
|
|
131
|
+
const TINY = 1e-300;
|
|
132
|
+
const EPSILON = 1e-15;
|
|
133
|
+
const MAX_ITERATIONS = 1e5;
|
|
134
|
+
const qab = a + b;
|
|
135
|
+
const qap = a + 1;
|
|
136
|
+
const qam = a - 1;
|
|
137
|
+
let c = 1;
|
|
138
|
+
let d = 1 - qab * x / qap;
|
|
139
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
140
|
+
d = 1 / d;
|
|
141
|
+
let h = d;
|
|
142
|
+
for (let m = 1; m <= MAX_ITERATIONS; m += 1) {
|
|
143
|
+
const m2 = 2 * m;
|
|
144
|
+
let aa = m * (b - m) * x / ((qam + m2) * (a + m2));
|
|
145
|
+
d = 1 + aa * d;
|
|
146
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
147
|
+
c = 1 + aa / c;
|
|
148
|
+
if (Math.abs(c) < TINY) c = TINY;
|
|
149
|
+
d = 1 / d;
|
|
150
|
+
h *= d * c;
|
|
151
|
+
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
|
|
152
|
+
d = 1 + aa * d;
|
|
153
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
154
|
+
c = 1 + aa / c;
|
|
155
|
+
if (Math.abs(c) < TINY) c = TINY;
|
|
156
|
+
d = 1 / d;
|
|
157
|
+
const delta = d * c;
|
|
158
|
+
h *= delta;
|
|
159
|
+
if (Math.abs(delta - 1) < EPSILON) break;
|
|
160
|
+
}
|
|
161
|
+
return h;
|
|
162
|
+
}
|
|
163
|
+
function regularizedIncompleteBeta(x, a, b) {
|
|
164
|
+
if (x <= 0) return 0;
|
|
165
|
+
if (x >= 1) return 1;
|
|
166
|
+
if (x > (a + 1) / (a + b + 2)) return 1 - regularizedIncompleteBeta(1 - x, b, a);
|
|
167
|
+
const logFront = a * Math.log(x) + b * Math.log1p(-x) - logBeta(a, b);
|
|
168
|
+
return Math.exp(logFront) * betaContinuedFraction(x, a, b) / a;
|
|
169
|
+
}
|
|
170
|
+
function probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB) {
|
|
171
|
+
const total = alphaA + betaA;
|
|
172
|
+
const meanA = alphaA / total;
|
|
173
|
+
const sdA = Math.sqrt(alphaA * betaA / (total * total * (total + 1)));
|
|
174
|
+
const low = Math.max(0, meanA - 40 * sdA);
|
|
175
|
+
const high = Math.min(1, meanA + 40 * sdA);
|
|
176
|
+
const center = (low + high) / 2;
|
|
177
|
+
const halfWidth = (high - low) / 2;
|
|
178
|
+
const logNormalizer = logBeta(alphaA, betaA);
|
|
179
|
+
const STEP = 1 / 128;
|
|
180
|
+
const T_MAX = 3.5;
|
|
181
|
+
let mass = 0;
|
|
182
|
+
let weighted = 0;
|
|
183
|
+
for (let t = -T_MAX; t <= T_MAX + 1e-12; t += STEP) {
|
|
184
|
+
const inner = Math.PI / 2 * Math.sinh(t);
|
|
185
|
+
const x = center + halfWidth * Math.tanh(inner);
|
|
186
|
+
if (x <= 0 || x >= 1) continue;
|
|
187
|
+
const coshInner = Math.cosh(inner);
|
|
188
|
+
const weight = Math.PI / 2 * Math.cosh(t) / (coshInner * coshInner);
|
|
189
|
+
const density = Math.exp((alphaA - 1) * Math.log(x) + (betaA - 1) * Math.log1p(-x) - logNormalizer);
|
|
190
|
+
if (!Number.isFinite(density)) continue;
|
|
191
|
+
mass += weight * density;
|
|
192
|
+
weighted += weight * density * (1 - regularizedIncompleteBeta(x, alphaB, betaB));
|
|
193
|
+
}
|
|
194
|
+
return mass > 0 ? weighted / mass : 0.5;
|
|
195
|
+
}
|
|
127
196
|
function inverseNormalCdf(p) {
|
|
128
197
|
const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
|
|
129
198
|
const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
|
|
@@ -158,6 +227,50 @@ function round(value, decimals = 6) {
|
|
|
158
227
|
return Math.round(value * factor) / factor;
|
|
159
228
|
}
|
|
160
229
|
|
|
230
|
+
// src/core/lift.ts
|
|
231
|
+
var LIFT_DENSITY_POINTS = 160;
|
|
232
|
+
var LIFT_INTEGRATION_STEPS = 400;
|
|
233
|
+
function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
|
|
234
|
+
const lower = quantile(sortedRelativeLifts, 2e-3);
|
|
235
|
+
const upper = quantile(sortedRelativeLifts, 0.998);
|
|
236
|
+
const span = upper - lower;
|
|
237
|
+
if (!Number.isFinite(span) || span <= 0) {
|
|
238
|
+
return [
|
|
239
|
+
{ relativeLift: lower, density: 0 },
|
|
240
|
+
{ relativeLift: lower, density: 1 },
|
|
241
|
+
{ relativeLift: lower, density: 0 }
|
|
242
|
+
];
|
|
243
|
+
}
|
|
244
|
+
const totalA = alphaA + betaA;
|
|
245
|
+
const meanA = alphaA / totalA;
|
|
246
|
+
const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
|
|
247
|
+
const aLow = Math.max(1e-9, meanA - 10 * sdA);
|
|
248
|
+
const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
|
|
249
|
+
const lifts = Array.from(
|
|
250
|
+
{ length: LIFT_DENSITY_POINTS },
|
|
251
|
+
(_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
|
|
252
|
+
);
|
|
253
|
+
const densities = lifts.map((relativeLift) => {
|
|
254
|
+
const ratio = 1 + relativeLift;
|
|
255
|
+
if (ratio <= 0) return 0;
|
|
256
|
+
const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
|
|
257
|
+
if (aHigh <= aLow) return 0;
|
|
258
|
+
const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
|
|
259
|
+
let total = 0;
|
|
260
|
+
for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
|
|
261
|
+
const a = aLow + step * index;
|
|
262
|
+
const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
|
|
263
|
+
total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
|
|
264
|
+
}
|
|
265
|
+
return total * step / 3;
|
|
266
|
+
});
|
|
267
|
+
const peak = Math.max(...densities, Number.MIN_VALUE);
|
|
268
|
+
return lifts.map((relativeLift, index) => ({
|
|
269
|
+
relativeLift,
|
|
270
|
+
density: densities[index] / peak
|
|
271
|
+
}));
|
|
272
|
+
}
|
|
273
|
+
|
|
161
274
|
// src/core/plan.ts
|
|
162
275
|
var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
|
|
163
276
|
function requiredSamplePerVariant({
|
|
@@ -204,6 +317,10 @@ function shareUrl(inputs) {
|
|
|
204
317
|
thresholdPercent: formatPercentParam(inputs.threshold),
|
|
205
318
|
meaningfulLiftPercent: formatPercentParam(inputs.minEffect)
|
|
206
319
|
});
|
|
320
|
+
if (inputs.prior && !(inputs.prior[0] === 1 && inputs.prior[1] === 1)) {
|
|
321
|
+
params.set("priorAlpha", String(inputs.prior[0]));
|
|
322
|
+
params.set("priorBeta", String(inputs.prior[1]));
|
|
323
|
+
}
|
|
207
324
|
return `${WEB_TOOL_URL}?${params.toString()}`;
|
|
208
325
|
}
|
|
209
326
|
|
|
@@ -215,13 +332,20 @@ var PRIOR_ALPHA = 1;
|
|
|
215
332
|
var PRIOR_BETA = 1;
|
|
216
333
|
var POSTERIOR_CHART_POINTS = 96;
|
|
217
334
|
var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
|
|
218
|
-
var WAITING_VALUE_SIMULATIONS =
|
|
335
|
+
var WAITING_VALUE_SIMULATIONS = 6e3;
|
|
219
336
|
var DECISION_LABELS = {
|
|
220
337
|
ship_b: "Ship Variant B",
|
|
221
338
|
keep_a: "Keep Variant A",
|
|
222
339
|
continue: "Continue test",
|
|
223
340
|
inconclusive: "Inconclusive"
|
|
224
341
|
};
|
|
342
|
+
function validatePrior(prior) {
|
|
343
|
+
const [alpha, beta] = prior;
|
|
344
|
+
if (!(Number.isFinite(alpha) && alpha > 0 && Number.isFinite(beta) && beta > 0)) {
|
|
345
|
+
throw new RangeError(`Prior parameters must be finite and greater than 0, got Beta(${alpha}, ${beta}).`);
|
|
346
|
+
}
|
|
347
|
+
return [alpha, beta];
|
|
348
|
+
}
|
|
225
349
|
function deriveSeed(...parts) {
|
|
226
350
|
let hash = 2166136261;
|
|
227
351
|
for (const part of parts) {
|
|
@@ -276,8 +400,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
|
|
|
276
400
|
});
|
|
277
401
|
}
|
|
278
402
|
function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
|
|
279
|
-
const
|
|
403
|
+
const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
|
|
280
404
|
return WAITING_SCENARIOS.map((extraPerVariant) => {
|
|
405
|
+
const random = makeDeterministicRandom(seed);
|
|
281
406
|
let regretSum = 0;
|
|
282
407
|
for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
|
|
283
408
|
const trueRateA = sampleBeta(alphaA, betaA, random);
|
|
@@ -303,12 +428,14 @@ function analyze({
|
|
|
303
428
|
minEffect = DEFAULT_MIN_EFFECT,
|
|
304
429
|
samples = DEFAULT_SAMPLES,
|
|
305
430
|
seed = null,
|
|
306
|
-
includeChart = false
|
|
431
|
+
includeChart = false,
|
|
432
|
+
prior = [PRIOR_ALPHA, PRIOR_BETA]
|
|
307
433
|
}) {
|
|
308
|
-
const
|
|
309
|
-
const
|
|
310
|
-
const
|
|
311
|
-
const
|
|
434
|
+
const [priorAlpha, priorBeta] = validatePrior(prior);
|
|
435
|
+
const alphaA = priorAlpha + conversionsA;
|
|
436
|
+
const betaA = priorBeta + (visitorsA - conversionsA);
|
|
437
|
+
const alphaB = priorAlpha + conversionsB;
|
|
438
|
+
const betaB = priorBeta + (visitorsB - conversionsB);
|
|
312
439
|
const rateA = conversionsA / visitorsA;
|
|
313
440
|
const rateB = conversionsB / visitorsB;
|
|
314
441
|
const posteriorMeanA = alphaA / (alphaA + betaA);
|
|
@@ -343,7 +470,7 @@ function analyze({
|
|
|
343
470
|
conversionsB,
|
|
344
471
|
threshold,
|
|
345
472
|
minEffect,
|
|
346
|
-
prior: [
|
|
473
|
+
prior: [priorAlpha, priorBeta],
|
|
347
474
|
samples,
|
|
348
475
|
seed: resolvedSeed
|
|
349
476
|
},
|
|
@@ -361,7 +488,9 @@ function analyze({
|
|
|
361
488
|
aBeatsB: 1 - pBBeatsA,
|
|
362
489
|
meaningfulLift: probabilityMeaningfulLift,
|
|
363
490
|
meaningfulHarm: probabilityMeaningfulHarm,
|
|
364
|
-
|
|
491
|
+
/* The exact complement of the closed-form win probability. Estimating it
|
|
492
|
+
from the samples left the two halves summing to slightly less than 1. */
|
|
493
|
+
harmful: 1 - pBBeatsA,
|
|
365
494
|
liftAbove1pct: share((lift) => lift > 0.01),
|
|
366
495
|
liftAbove5pct: share((lift) => lift > 0.05),
|
|
367
496
|
harmBeyond5pct: share((lift) => lift < -0.05),
|
|
@@ -373,7 +502,9 @@ function analyze({
|
|
|
373
502
|
absoluteDiff: posteriorMeanB - posteriorMeanA,
|
|
374
503
|
absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
|
|
375
504
|
observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
|
|
376
|
-
observedAbsoluteDiff: rateB - rateA
|
|
505
|
+
observedAbsoluteDiff: rateB - rateA,
|
|
506
|
+
relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
|
|
507
|
+
relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
|
|
377
508
|
},
|
|
378
509
|
expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
|
|
379
510
|
variants: {
|
|
@@ -387,12 +518,13 @@ function analyze({
|
|
|
387
518
|
},
|
|
388
519
|
waiting: waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow),
|
|
389
520
|
frequentist: { pValueTwoSided: twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) },
|
|
390
|
-
shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect })
|
|
521
|
+
shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] })
|
|
391
522
|
};
|
|
392
523
|
if (includeChart) {
|
|
393
524
|
const sortedA = [...samplesA].sort((left, right) => left - right);
|
|
394
525
|
const sortedB = [...samplesB].sort((left, right) => left - right);
|
|
395
526
|
result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
|
|
527
|
+
result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
|
|
396
528
|
}
|
|
397
529
|
return result;
|
|
398
530
|
}
|
|
@@ -474,9 +606,11 @@ function project({
|
|
|
474
606
|
stopRule = "lift",
|
|
475
607
|
maxLoss = DEFAULT_MAX_LOSS,
|
|
476
608
|
simulations = DEFAULT_SIMULATIONS,
|
|
477
|
-
seed = null
|
|
609
|
+
seed = null,
|
|
610
|
+
prior = [PRIOR_ALPHA, PRIOR_BETA]
|
|
478
611
|
}) {
|
|
479
|
-
const
|
|
612
|
+
const [priorAlpha, priorBeta] = validatePrior(prior);
|
|
613
|
+
const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
|
|
480
614
|
const resolvedSeed = seed ?? deriveSeed(visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, ["lift", "superiority", "loss"].indexOf(stopRule), simulations);
|
|
481
615
|
const inputs = {
|
|
482
616
|
visitorsA,
|
|
@@ -485,7 +619,7 @@ function project({
|
|
|
485
619
|
conversionsB,
|
|
486
620
|
threshold,
|
|
487
621
|
minEffect,
|
|
488
|
-
prior: [
|
|
622
|
+
prior: [priorAlpha, priorBeta],
|
|
489
623
|
dailyVisitors,
|
|
490
624
|
horizonDays,
|
|
491
625
|
stopRule,
|
|
@@ -493,7 +627,7 @@ function project({
|
|
|
493
627
|
simulations,
|
|
494
628
|
seed: resolvedSeed
|
|
495
629
|
};
|
|
496
|
-
const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
|
|
630
|
+
const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
|
|
497
631
|
const diagnostic = stopRule === "lift" && liftNearThreshold(current, minEffect) ? "lift_near_threshold" : null;
|
|
498
632
|
const decisionNow = decisionNowFrom(current, stopRule, threshold, maxLoss);
|
|
499
633
|
if (decisionNow) {
|
|
@@ -510,10 +644,10 @@ function project({
|
|
|
510
644
|
shareUrl: link
|
|
511
645
|
};
|
|
512
646
|
}
|
|
513
|
-
const alphaA0 =
|
|
514
|
-
const betaA0 =
|
|
515
|
-
const alphaB0 =
|
|
516
|
-
const betaB0 =
|
|
647
|
+
const alphaA0 = priorAlpha + conversionsA;
|
|
648
|
+
const betaA0 = priorBeta + (visitorsA - conversionsA);
|
|
649
|
+
const alphaB0 = priorAlpha + conversionsB;
|
|
650
|
+
const betaB0 = priorBeta + (visitorsB - conversionsB);
|
|
517
651
|
const random = makeDeterministicRandom(resolvedSeed);
|
|
518
652
|
const decidedByDay = new Array(horizonDays + 1).fill(0);
|
|
519
653
|
const decisionDays = [];
|
|
@@ -593,6 +727,8 @@ var OPTIONS = {
|
|
|
593
727
|
"rate-b": { type: "string" },
|
|
594
728
|
threshold: { type: "string" },
|
|
595
729
|
"min-effect": { type: "string" },
|
|
730
|
+
"prior-alpha": { type: "string" },
|
|
731
|
+
"prior-beta": { type: "string" },
|
|
596
732
|
"daily-visitors": { type: "string" },
|
|
597
733
|
horizon: { type: "string" },
|
|
598
734
|
"stop-rule": { type: "string" },
|
|
@@ -683,10 +819,19 @@ function parseMinEffect(values) {
|
|
|
683
819
|
if (minEffect < 0 || minEffect >= 1) throw new CliError("--min-effect must be a percent between 0 and 100.", { field: "min-effect" });
|
|
684
820
|
return minEffect;
|
|
685
821
|
}
|
|
822
|
+
function parsePriorParameter(flag, value, fallback) {
|
|
823
|
+
if (value === void 0) return fallback;
|
|
824
|
+
const parsed = parseNumber(flag, value);
|
|
825
|
+
if (parsed <= 0) throw new CliError(`--${flag} must be greater than 0, got "${value}".`, { field: flag });
|
|
826
|
+
return parsed;
|
|
827
|
+
}
|
|
828
|
+
function parsePrior(values) {
|
|
829
|
+
return [parsePriorParameter("prior-alpha", values["prior-alpha"], PRIOR_ALPHA), parsePriorParameter("prior-beta", values["prior-beta"], PRIOR_BETA)];
|
|
830
|
+
}
|
|
686
831
|
function parseAnalyzeArgs(values) {
|
|
687
832
|
const samples = values.samples === void 0 ? DEFAULT_SAMPLES : parseCount("samples", values.samples, { min: 100 });
|
|
688
833
|
const seed = values.seed === void 0 ? null : parseCount("seed", values.seed);
|
|
689
|
-
return { ...parseVariants(values), threshold: parseThreshold(values), minEffect: parseMinEffect(values), samples, seed };
|
|
834
|
+
return { ...parseVariants(values), threshold: parseThreshold(values), minEffect: parseMinEffect(values), prior: parsePrior(values), samples, seed };
|
|
690
835
|
}
|
|
691
836
|
function parseStopRule(value) {
|
|
692
837
|
if (value === void 0) return "lift";
|
|
@@ -762,7 +907,8 @@ function helpText(version) {
|
|
|
762
907
|
return `bayes-ab ${version} Bayesian A/B test analysis for humans and agents
|
|
763
908
|
|
|
764
909
|
Compares two variants of a binary outcome (signup, click, purchase) using
|
|
765
|
-
Beta
|
|
910
|
+
Beta posteriors on each conversion rate, from a flat Beta(1,1) prior unless
|
|
911
|
+
you choose another. Reports the probability that B
|
|
766
912
|
beats A, the probability the lift clears a meaningful threshold, expected
|
|
767
913
|
loss from shipping either variant, and a forward projection of how long the
|
|
768
914
|
test needs to run. Same math as ${WEB_TOOL_URL}
|
|
@@ -789,6 +935,12 @@ INPUTS (analyze, project)
|
|
|
789
935
|
--min-effect PCT Meaningful relative effect, symmetric: with 10, B must be
|
|
790
936
|
10% better than A to count as a win or 10% worse to count
|
|
791
937
|
as a loss. Relative, not percentage points. default 10
|
|
938
|
+
--prior-alpha N, --prior-beta N Beta(alpha, beta) prior on each variant's conversion rate,
|
|
939
|
+
the same for both. Any positive number. Reads as alpha
|
|
940
|
+
conversions in alpha + beta visitors of prior evidence,
|
|
941
|
+
with prior mean alpha / (alpha + beta).
|
|
942
|
+
Beta(1,1) is flat; Beta(0.5,0.5) is Jeffreys; Beta(55,945)
|
|
943
|
+
says "about 5.5%, worth 1,000 visitors". default 1, 1
|
|
792
944
|
Whole numbers may include commas or underscores: 12,000 and 12_000 both work.
|
|
793
945
|
|
|
794
946
|
PROJECTION (project)
|
|
@@ -819,7 +971,7 @@ OUTPUT
|
|
|
819
971
|
so piping or calling from an agent gets JSON automatically.
|
|
820
972
|
--pretty Indent the JSON.
|
|
821
973
|
--brief One-line human summary (analyze and project).
|
|
822
|
-
--include-chart Add
|
|
974
|
+
--include-chart Add posterior and lift density curves to analyze's JSON.
|
|
823
975
|
--samples N Monte Carlo samples for intervals and loss. default 12000
|
|
824
976
|
--seed N RNG seed. Defaults to a value derived from the inputs, so identical
|
|
825
977
|
inputs always give identical output.
|
|
@@ -884,6 +1036,10 @@ function unsignedPoints(fraction2, digits = 4) {
|
|
|
884
1036
|
function count(value) {
|
|
885
1037
|
return Math.round(value).toLocaleString("en-US");
|
|
886
1038
|
}
|
|
1039
|
+
function betaParameter(value) {
|
|
1040
|
+
if (Number.isInteger(value)) return count(value);
|
|
1041
|
+
return value.toLocaleString("en-US", { maximumFractionDigits: 3 });
|
|
1042
|
+
}
|
|
887
1043
|
function missedConversions(regretRate, visitors = 1e4) {
|
|
888
1044
|
const missed = regretRate * visitors;
|
|
889
1045
|
const digits = missed < 1 ? 2 : missed < 10 ? 1 : 0;
|
|
@@ -937,6 +1093,8 @@ function renderAnalyzeBrief(result, derivedFrom2) {
|
|
|
937
1093
|
`lift ${signedPercent(result.effect.relativeLift)} (CI ${signedPercent(lower)} to ${signedPercent(upper)})`,
|
|
938
1094
|
`loss if ship B ${unsignedPoints(result.expectedLoss.shipB)}`
|
|
939
1095
|
];
|
|
1096
|
+
const [priorAlpha, priorBeta] = result.inputs.prior;
|
|
1097
|
+
if (priorAlpha !== 1 || priorBeta !== 1) parts.push(`Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) prior`);
|
|
940
1098
|
const note = derivedNote(result, derivedFrom2);
|
|
941
1099
|
return parts.join(" \xB7 ") + (note ? ` ${note}` : "");
|
|
942
1100
|
}
|
|
@@ -948,7 +1106,7 @@ function renderAnalyze(result, derivedFrom2) {
|
|
|
948
1106
|
const lines = [];
|
|
949
1107
|
const note = derivedNote(result, derivedFrom2);
|
|
950
1108
|
lines.push(
|
|
951
|
-
`Bayesian A/B test Beta(${inputs.prior[0]},${inputs.prior[1]}) prior \xB7 decision threshold ${percent(inputs.threshold, 0)} \xB7 meaningful effect \xB1${minEffect} relative`
|
|
1109
|
+
`Bayesian A/B test Beta(${betaParameter(inputs.prior[0])},${betaParameter(inputs.prior[1])}) prior \xB7 decision threshold ${percent(inputs.threshold, 0)} \xB7 meaningful effect \xB1${minEffect} relative`
|
|
952
1110
|
);
|
|
953
1111
|
if (note) lines.push(` ${note}`);
|
|
954
1112
|
lines.push("");
|
|
@@ -974,7 +1132,7 @@ function renderAnalyze(result, derivedFrom2) {
|
|
|
974
1132
|
["Conversions", count(variants.A.conversions), count(variants.B.conversions)],
|
|
975
1133
|
["Observed rate", percent(variants.A.rate), percent(variants.B.rate)],
|
|
976
1134
|
["Posterior mean", percent(variants.A.posteriorMean, 3), percent(variants.B.posteriorMean, 3)],
|
|
977
|
-
["Posterior", `Beta(${
|
|
1135
|
+
["Posterior", `Beta(${betaParameter(variants.A.posterior[0])}, ${betaParameter(variants.A.posterior[1])})`, `Beta(${betaParameter(variants.B.posterior[0])}, ${betaParameter(variants.B.posterior[1])})`]
|
|
978
1136
|
])
|
|
979
1137
|
);
|
|
980
1138
|
lines.push("");
|
|
@@ -1112,6 +1270,8 @@ function renderProjectBrief(result) {
|
|
|
1112
1270
|
result.daysToDecision ? `median day ${result.daysToDecision.p50} if it decides (p90 day ${result.daysToDecision.p90})` : "no simulated future decided",
|
|
1113
1271
|
`${count(dailyVisitors)}/day per variant`
|
|
1114
1272
|
];
|
|
1273
|
+
const [priorAlpha, priorBeta] = result.inputs.prior;
|
|
1274
|
+
if (priorAlpha !== 1 || priorBeta !== 1) parts.push(`Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) prior`);
|
|
1115
1275
|
return parts.join(" \xB7 ");
|
|
1116
1276
|
}
|
|
1117
1277
|
function renderProject(result) {
|
|
@@ -1121,6 +1281,8 @@ function renderProject(result) {
|
|
|
1121
1281
|
`Forward projection ${count(inputs.dailyVisitors)} visitors/day per variant \xB7 ${inputs.horizonDays}-day horizon \xB7 ${count(inputs.simulations)} simulated futures`
|
|
1122
1282
|
);
|
|
1123
1283
|
lines.push(`Stopping rule ${describeRule(result)}`);
|
|
1284
|
+
const [priorAlpha, priorBeta] = inputs.prior;
|
|
1285
|
+
if (priorAlpha !== 1 || priorBeta !== 1) lines.push(`Prior Beta(${betaParameter(priorAlpha)},${betaParameter(priorBeta)}) on each variant`);
|
|
1124
1286
|
lines.push("");
|
|
1125
1287
|
if (result.alreadyDecided && result.decisionNow) {
|
|
1126
1288
|
lines.push(`Already decided ${decisionLabel(result.decisionNow)}`);
|
|
@@ -1186,7 +1348,7 @@ var variantInputs = {
|
|
|
1186
1348
|
conversionsB: integer,
|
|
1187
1349
|
threshold: fraction,
|
|
1188
1350
|
minEffect: fraction,
|
|
1189
|
-
prior: pair(number)
|
|
1351
|
+
prior: { ...pair({ type: "number", exclusiveMinimum: 0 }), description: "The Beta(alpha, beta) prior applied to each variant. [1, 1] unless --prior-alpha or --prior-beta was given." }
|
|
1190
1352
|
};
|
|
1191
1353
|
var derivedFrom = {
|
|
1192
1354
|
...object({ rateA: fraction, rateB: fraction }, []),
|
|
@@ -1217,7 +1379,7 @@ var ANALYZE_SCHEMA = {
|
|
|
1217
1379
|
aBeatsB: fraction,
|
|
1218
1380
|
meaningfulLift: { ...fraction, description: "P(relative lift > minEffect)" },
|
|
1219
1381
|
meaningfulHarm: { ...fraction, description: "P(relative lift < -minEffect)" },
|
|
1220
|
-
harmful: { ...fraction, description: "P(
|
|
1382
|
+
harmful: { ...fraction, description: "P(B < A), the exact complement of bBeatsA" },
|
|
1221
1383
|
liftAbove1pct: fraction,
|
|
1222
1384
|
liftAbove5pct: fraction,
|
|
1223
1385
|
harmBeyond5pct: fraction,
|
|
@@ -1229,7 +1391,9 @@ var ANALYZE_SCHEMA = {
|
|
|
1229
1391
|
absoluteDiff: number,
|
|
1230
1392
|
absoluteDiffCI95: pair(number),
|
|
1231
1393
|
observedRelativeLift: { type: ["number", "null"] },
|
|
1232
|
-
observedAbsoluteDiff: number
|
|
1394
|
+
observedAbsoluteDiff: number,
|
|
1395
|
+
relativeLiftMedian: number,
|
|
1396
|
+
relativeLiftIQR: pair(number)
|
|
1233
1397
|
}),
|
|
1234
1398
|
expectedLoss: object({ shipA: fraction, shipB: fraction }),
|
|
1235
1399
|
variants: object({
|
|
@@ -1251,6 +1415,11 @@ var ANALYZE_SCHEMA = {
|
|
|
1251
1415
|
description: "Only with --include-chart.",
|
|
1252
1416
|
items: object({ rate: fraction, densityA: number, densityB: number })
|
|
1253
1417
|
},
|
|
1418
|
+
liftChart: {
|
|
1419
|
+
type: "array",
|
|
1420
|
+
description: "Only with --include-chart. Exact density of the relative lift, peak scaled to 1.",
|
|
1421
|
+
items: object({ relativeLift: number, density: number })
|
|
1422
|
+
},
|
|
1254
1423
|
shareUrl: { type: "string", format: "uri" }
|
|
1255
1424
|
}, [
|
|
1256
1425
|
"tool",
|
|
@@ -1343,7 +1512,7 @@ var SCHEMAS = {
|
|
|
1343
1512
|
};
|
|
1344
1513
|
|
|
1345
1514
|
// src/cli/main.ts
|
|
1346
|
-
var VERSION = true ? "0.
|
|
1515
|
+
var VERSION = true ? "0.3.0" : "dev";
|
|
1347
1516
|
var JSON_DECIMALS = 6;
|
|
1348
1517
|
function roundDeep(value) {
|
|
1349
1518
|
if (typeof value === "number") return round(value, JSON_DECIMALS);
|
|
@@ -1367,7 +1536,7 @@ function envelope2(command, result, derivedFrom2) {
|
|
|
1367
1536
|
return { tool: "bayes-ab", version: VERSION, command, ...result, inputs };
|
|
1368
1537
|
}
|
|
1369
1538
|
function stripChart(result) {
|
|
1370
|
-
const { chart, ...rest } = result;
|
|
1539
|
+
const { chart, liftChart, ...rest } = result;
|
|
1371
1540
|
return rest;
|
|
1372
1541
|
}
|
|
1373
1542
|
function run(argv, io) {
|
package/dist/core/analyze.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type LiftDensityPoint } from "./lift.ts";
|
|
1
2
|
export declare const DEFAULT_THRESHOLD = 0.95;
|
|
2
3
|
export declare const DEFAULT_MIN_EFFECT = 0.1;
|
|
3
4
|
export declare const DEFAULT_SAMPLES = 12000;
|
|
@@ -18,6 +19,8 @@ export type AnalyzeInputs = {
|
|
|
18
19
|
samples?: number;
|
|
19
20
|
seed?: number | null;
|
|
20
21
|
includeChart?: boolean;
|
|
22
|
+
/** Beta(alpha, beta) prior applied to both variants. Defaults to the flat Beta(1, 1). */
|
|
23
|
+
prior?: [number, number];
|
|
21
24
|
};
|
|
22
25
|
export type VariantSummary = {
|
|
23
26
|
visitors: number;
|
|
@@ -75,6 +78,8 @@ export type AnalyzeResult = {
|
|
|
75
78
|
absoluteDiffCI95: [number, number];
|
|
76
79
|
observedRelativeLift: number | null;
|
|
77
80
|
observedAbsoluteDiff: number;
|
|
81
|
+
relativeLiftMedian: number;
|
|
82
|
+
relativeLiftIQR: [number, number];
|
|
78
83
|
};
|
|
79
84
|
expectedLoss: {
|
|
80
85
|
shipA: number;
|
|
@@ -94,8 +99,12 @@ export type AnalyzeResult = {
|
|
|
94
99
|
pValueTwoSided: number;
|
|
95
100
|
};
|
|
96
101
|
chart?: ChartPoint[];
|
|
102
|
+
liftChart?: LiftDensityPoint[];
|
|
97
103
|
shareUrl: string;
|
|
98
104
|
};
|
|
105
|
+
/** Throws a RangeError unless both prior parameters are finite and positive. */
|
|
106
|
+
export declare function validatePrior(prior: [number, number]): [number, number];
|
|
107
|
+
export declare function isDefaultPrior([alpha, beta]: [number, number]): boolean;
|
|
99
108
|
export declare function deriveSeed(...parts: number[]): number;
|
|
100
109
|
export declare function decisionStatus({ probabilityMeaningfulLift, probabilityMeaningfulHarm, threshold, evidence, }: {
|
|
101
110
|
probabilityMeaningfulLift: number;
|
|
@@ -103,4 +112,4 @@ export declare function decisionStatus({ probabilityMeaningfulLift, probabilityM
|
|
|
103
112
|
threshold: number;
|
|
104
113
|
evidence: EvidenceStrength;
|
|
105
114
|
}): DecisionStatus;
|
|
106
|
-
export declare function analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, samples, seed, includeChart, }: AnalyzeInputs): AnalyzeResult;
|
|
115
|
+
export declare function analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, samples, seed, includeChart, prior, }: AnalyzeInputs): AnalyzeResult;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type LiftDensityPoint = {
|
|
2
|
+
relativeLift: number;
|
|
3
|
+
/** Scaled so the peak is 1. */
|
|
4
|
+
density: number;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Density of B's relative lift over A, computed exactly rather than
|
|
8
|
+
* histogrammed from the Monte Carlo samples. Writing B = A(1 + r), the ratio
|
|
9
|
+
* density is f(r) = ∫ a · f_A(a) · f_B(a(1+r)) da, which is one Simpson
|
|
10
|
+
* integral per grid point. A histogram of 12k draws carries ~10% sampling
|
|
11
|
+
* noise per bin, and on a wide posterior that reads as a jagged curve.
|
|
12
|
+
*
|
|
13
|
+
* The sorted Monte Carlo lifts only set the plotted range (0.2% to 99.8%).
|
|
14
|
+
*/
|
|
15
|
+
export declare function liftDensity(alphaA: number, betaA: number, alphaB: number, betaB: number, sortedRelativeLifts: number[]): LiftDensityPoint[];
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Expected loss of each choice by direct numerical integration over a 600-step
|
|
3
|
+
* grid, with no sampling. Deterministic and smooth in its inputs, so it suits
|
|
4
|
+
* curves that sweep sample size; `analyze` uses Monte Carlo instead because it
|
|
5
|
+
* shares those draws with the lift interval and threshold probabilities.
|
|
6
|
+
*
|
|
7
|
+
* Losses are absolute conversion-rate differences (0.001 = 0.1 pp):
|
|
8
|
+
* shipA = E[max(B − A, 0)], shipB = E[max(A − B, 0)].
|
|
9
|
+
*/
|
|
10
|
+
export declare function integratedExpectedLoss(alphaA: number, betaA: number, alphaB: number, betaB: number): {
|
|
11
|
+
shipA: number;
|
|
12
|
+
shipB: number;
|
|
13
|
+
};
|
package/dist/core/math.d.ts
CHANGED
|
@@ -10,8 +10,16 @@ export declare function sampleBeta(alpha: number, beta: number, random: Random):
|
|
|
10
10
|
export declare function sampleBinomialApprox(trials: number, probability: number, random: Random): number;
|
|
11
11
|
export declare function quantile(sortedValues: number[], probability: number): number;
|
|
12
12
|
export declare function mean(values: number[]): number;
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* P(B > A) for independent Beta posteriors. Exact via Evan Miller's closed form
|
|
15
|
+
* when the alphas are whole numbers, which they are under any whole-number
|
|
16
|
+
* prior. The closed form sums over 0..alpha-1, so a fractional alpha (a
|
|
17
|
+
* Jeffreys Beta(0.5, 0.5) prior, say) falls back to numerical integration,
|
|
18
|
+
* accurate to roughly 1e-10.
|
|
19
|
+
*/
|
|
14
20
|
export declare function probabilityBBeatsA(alphaA: number, betaA: number, alphaB: number, betaB: number): number;
|
|
21
|
+
/** Regularized incomplete beta function I_x(a, b): the Beta(a, b) CDF at x. */
|
|
22
|
+
export declare function regularizedIncompleteBeta(x: number, a: number, b: number): number;
|
|
15
23
|
export declare function inverseNormalCdf(p: number): number;
|
|
16
24
|
export declare function erf(x: number): number;
|
|
17
25
|
export declare function normalCdf(x: number): number;
|
package/dist/core/project.d.ts
CHANGED
|
@@ -16,6 +16,8 @@ export type ProjectInputs = {
|
|
|
16
16
|
maxLoss?: number;
|
|
17
17
|
simulations?: number;
|
|
18
18
|
seed?: number | null;
|
|
19
|
+
/** Beta(alpha, beta) prior applied to both variants. Defaults to the flat Beta(1, 1). */
|
|
20
|
+
prior?: [number, number];
|
|
19
21
|
};
|
|
20
22
|
export type ProjectResult = {
|
|
21
23
|
inputs: {
|
|
@@ -55,4 +57,4 @@ export type ProjectResult = {
|
|
|
55
57
|
approximation: "normal";
|
|
56
58
|
shareUrl: string;
|
|
57
59
|
};
|
|
58
|
-
export declare function project({ visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, threshold, minEffect, stopRule, maxLoss, simulations, seed, }: ProjectInputs): ProjectResult;
|
|
60
|
+
export declare function project({ visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, threshold, minEffect, stopRule, maxLoss, simulations, seed, prior, }: ProjectInputs): ProjectResult;
|
package/dist/core/share.d.ts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD } from "./core/analyze.ts";
|
|
1
|
+
export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD, PRIOR_ALPHA, PRIOR_BETA, isDefaultPrior, validatePrior } from "./core/analyze.ts";
|
|
2
2
|
export type { AnalyzeInputs, AnalyzeResult, ChartPoint, DecisionStatus, EvidenceStrength, Maturity, Variant, VariantSummary, WaitingScenario, } from "./core/analyze.ts";
|
|
3
3
|
export { project, DEFAULT_HORIZON_DAYS, DEFAULT_MAX_LOSS, DEFAULT_SIMULATIONS } from "./core/project.ts";
|
|
4
4
|
export type { ProjectInputs, ProjectResult, ProjectedDecision, StopRule } from "./core/project.ts";
|
|
5
5
|
export { plan, requiredSamplePerVariant } from "./core/plan.ts";
|
|
6
6
|
export type { PlanInputs, PlanResult, PlanRow } from "./core/plan.ts";
|
|
7
7
|
export { shareUrl, WEB_TOOL_URL } from "./core/share.ts";
|
|
8
|
-
export { probabilityBBeatsA } from "./core/math.ts";
|
|
8
|
+
export { betaPdf, logBeta, logGamma, probabilityBBeatsA, regularizedIncompleteBeta } from "./core/math.ts";
|
|
9
|
+
export { liftDensity } from "./core/lift.ts";
|
|
10
|
+
export type { LiftDensityPoint } from "./core/lift.ts";
|
|
11
|
+
export { integratedExpectedLoss } from "./core/loss.ts";
|
package/dist/index.js
CHANGED
|
@@ -117,11 +117,80 @@ function probabilityRightBeatsLeftDirect(alphaLeft, betaLeft, alphaRight, betaRi
|
|
|
117
117
|
return Math.exp(maxLog) * scaledSum;
|
|
118
118
|
}
|
|
119
119
|
function probabilityBBeatsA(alphaA, betaA, alphaB, betaB) {
|
|
120
|
+
if (!Number.isInteger(alphaA) || !Number.isInteger(alphaB)) {
|
|
121
|
+
return probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB);
|
|
122
|
+
}
|
|
120
123
|
if (alphaB <= alphaA) {
|
|
121
124
|
return probabilityRightBeatsLeftDirect(alphaA, betaA, alphaB, betaB);
|
|
122
125
|
}
|
|
123
126
|
return 1 - probabilityRightBeatsLeftDirect(alphaB, betaB, alphaA, betaA);
|
|
124
127
|
}
|
|
128
|
+
function betaContinuedFraction(x, a, b) {
|
|
129
|
+
const TINY = 1e-300;
|
|
130
|
+
const EPSILON = 1e-15;
|
|
131
|
+
const MAX_ITERATIONS = 1e5;
|
|
132
|
+
const qab = a + b;
|
|
133
|
+
const qap = a + 1;
|
|
134
|
+
const qam = a - 1;
|
|
135
|
+
let c = 1;
|
|
136
|
+
let d = 1 - qab * x / qap;
|
|
137
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
138
|
+
d = 1 / d;
|
|
139
|
+
let h = d;
|
|
140
|
+
for (let m = 1; m <= MAX_ITERATIONS; m += 1) {
|
|
141
|
+
const m2 = 2 * m;
|
|
142
|
+
let aa = m * (b - m) * x / ((qam + m2) * (a + m2));
|
|
143
|
+
d = 1 + aa * d;
|
|
144
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
145
|
+
c = 1 + aa / c;
|
|
146
|
+
if (Math.abs(c) < TINY) c = TINY;
|
|
147
|
+
d = 1 / d;
|
|
148
|
+
h *= d * c;
|
|
149
|
+
aa = -(a + m) * (qab + m) * x / ((a + m2) * (qap + m2));
|
|
150
|
+
d = 1 + aa * d;
|
|
151
|
+
if (Math.abs(d) < TINY) d = TINY;
|
|
152
|
+
c = 1 + aa / c;
|
|
153
|
+
if (Math.abs(c) < TINY) c = TINY;
|
|
154
|
+
d = 1 / d;
|
|
155
|
+
const delta = d * c;
|
|
156
|
+
h *= delta;
|
|
157
|
+
if (Math.abs(delta - 1) < EPSILON) break;
|
|
158
|
+
}
|
|
159
|
+
return h;
|
|
160
|
+
}
|
|
161
|
+
function regularizedIncompleteBeta(x, a, b) {
|
|
162
|
+
if (x <= 0) return 0;
|
|
163
|
+
if (x >= 1) return 1;
|
|
164
|
+
if (x > (a + 1) / (a + b + 2)) return 1 - regularizedIncompleteBeta(1 - x, b, a);
|
|
165
|
+
const logFront = a * Math.log(x) + b * Math.log1p(-x) - logBeta(a, b);
|
|
166
|
+
return Math.exp(logFront) * betaContinuedFraction(x, a, b) / a;
|
|
167
|
+
}
|
|
168
|
+
function probabilityBBeatsANumeric(alphaA, betaA, alphaB, betaB) {
|
|
169
|
+
const total = alphaA + betaA;
|
|
170
|
+
const meanA = alphaA / total;
|
|
171
|
+
const sdA = Math.sqrt(alphaA * betaA / (total * total * (total + 1)));
|
|
172
|
+
const low = Math.max(0, meanA - 40 * sdA);
|
|
173
|
+
const high = Math.min(1, meanA + 40 * sdA);
|
|
174
|
+
const center = (low + high) / 2;
|
|
175
|
+
const halfWidth = (high - low) / 2;
|
|
176
|
+
const logNormalizer = logBeta(alphaA, betaA);
|
|
177
|
+
const STEP = 1 / 128;
|
|
178
|
+
const T_MAX = 3.5;
|
|
179
|
+
let mass = 0;
|
|
180
|
+
let weighted = 0;
|
|
181
|
+
for (let t = -T_MAX; t <= T_MAX + 1e-12; t += STEP) {
|
|
182
|
+
const inner = Math.PI / 2 * Math.sinh(t);
|
|
183
|
+
const x = center + halfWidth * Math.tanh(inner);
|
|
184
|
+
if (x <= 0 || x >= 1) continue;
|
|
185
|
+
const coshInner = Math.cosh(inner);
|
|
186
|
+
const weight = Math.PI / 2 * Math.cosh(t) / (coshInner * coshInner);
|
|
187
|
+
const density = Math.exp((alphaA - 1) * Math.log(x) + (betaA - 1) * Math.log1p(-x) - logNormalizer);
|
|
188
|
+
if (!Number.isFinite(density)) continue;
|
|
189
|
+
mass += weight * density;
|
|
190
|
+
weighted += weight * density * (1 - regularizedIncompleteBeta(x, alphaB, betaB));
|
|
191
|
+
}
|
|
192
|
+
return mass > 0 ? weighted / mass : 0.5;
|
|
193
|
+
}
|
|
125
194
|
function inverseNormalCdf(p) {
|
|
126
195
|
const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
|
|
127
196
|
const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
|
|
@@ -151,6 +220,50 @@ function normalPdf(x) {
|
|
|
151
220
|
return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
|
|
152
221
|
}
|
|
153
222
|
|
|
223
|
+
// src/core/lift.ts
|
|
224
|
+
var LIFT_DENSITY_POINTS = 160;
|
|
225
|
+
var LIFT_INTEGRATION_STEPS = 400;
|
|
226
|
+
function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
|
|
227
|
+
const lower = quantile(sortedRelativeLifts, 2e-3);
|
|
228
|
+
const upper = quantile(sortedRelativeLifts, 0.998);
|
|
229
|
+
const span = upper - lower;
|
|
230
|
+
if (!Number.isFinite(span) || span <= 0) {
|
|
231
|
+
return [
|
|
232
|
+
{ relativeLift: lower, density: 0 },
|
|
233
|
+
{ relativeLift: lower, density: 1 },
|
|
234
|
+
{ relativeLift: lower, density: 0 }
|
|
235
|
+
];
|
|
236
|
+
}
|
|
237
|
+
const totalA = alphaA + betaA;
|
|
238
|
+
const meanA = alphaA / totalA;
|
|
239
|
+
const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
|
|
240
|
+
const aLow = Math.max(1e-9, meanA - 10 * sdA);
|
|
241
|
+
const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
|
|
242
|
+
const lifts = Array.from(
|
|
243
|
+
{ length: LIFT_DENSITY_POINTS },
|
|
244
|
+
(_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
|
|
245
|
+
);
|
|
246
|
+
const densities = lifts.map((relativeLift) => {
|
|
247
|
+
const ratio = 1 + relativeLift;
|
|
248
|
+
if (ratio <= 0) return 0;
|
|
249
|
+
const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
|
|
250
|
+
if (aHigh <= aLow) return 0;
|
|
251
|
+
const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
|
|
252
|
+
let total = 0;
|
|
253
|
+
for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
|
|
254
|
+
const a = aLow + step * index;
|
|
255
|
+
const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
|
|
256
|
+
total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
|
|
257
|
+
}
|
|
258
|
+
return total * step / 3;
|
|
259
|
+
});
|
|
260
|
+
const peak = Math.max(...densities, Number.MIN_VALUE);
|
|
261
|
+
return lifts.map((relativeLift, index) => ({
|
|
262
|
+
relativeLift,
|
|
263
|
+
density: densities[index] / peak
|
|
264
|
+
}));
|
|
265
|
+
}
|
|
266
|
+
|
|
154
267
|
// src/core/plan.ts
|
|
155
268
|
var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
|
|
156
269
|
function requiredSamplePerVariant({
|
|
@@ -197,6 +310,10 @@ function shareUrl(inputs) {
|
|
|
197
310
|
thresholdPercent: formatPercentParam(inputs.threshold),
|
|
198
311
|
meaningfulLiftPercent: formatPercentParam(inputs.minEffect)
|
|
199
312
|
});
|
|
313
|
+
if (inputs.prior && !(inputs.prior[0] === 1 && inputs.prior[1] === 1)) {
|
|
314
|
+
params.set("priorAlpha", String(inputs.prior[0]));
|
|
315
|
+
params.set("priorBeta", String(inputs.prior[1]));
|
|
316
|
+
}
|
|
200
317
|
return `${WEB_TOOL_URL}?${params.toString()}`;
|
|
201
318
|
}
|
|
202
319
|
|
|
@@ -208,13 +325,23 @@ var PRIOR_ALPHA = 1;
|
|
|
208
325
|
var PRIOR_BETA = 1;
|
|
209
326
|
var POSTERIOR_CHART_POINTS = 96;
|
|
210
327
|
var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
|
|
211
|
-
var WAITING_VALUE_SIMULATIONS =
|
|
328
|
+
var WAITING_VALUE_SIMULATIONS = 6e3;
|
|
212
329
|
var DECISION_LABELS = {
|
|
213
330
|
ship_b: "Ship Variant B",
|
|
214
331
|
keep_a: "Keep Variant A",
|
|
215
332
|
continue: "Continue test",
|
|
216
333
|
inconclusive: "Inconclusive"
|
|
217
334
|
};
|
|
335
|
+
function validatePrior(prior) {
|
|
336
|
+
const [alpha, beta] = prior;
|
|
337
|
+
if (!(Number.isFinite(alpha) && alpha > 0 && Number.isFinite(beta) && beta > 0)) {
|
|
338
|
+
throw new RangeError(`Prior parameters must be finite and greater than 0, got Beta(${alpha}, ${beta}).`);
|
|
339
|
+
}
|
|
340
|
+
return [alpha, beta];
|
|
341
|
+
}
|
|
342
|
+
function isDefaultPrior([alpha, beta]) {
|
|
343
|
+
return alpha === PRIOR_ALPHA && beta === PRIOR_BETA;
|
|
344
|
+
}
|
|
218
345
|
function deriveSeed(...parts) {
|
|
219
346
|
let hash = 2166136261;
|
|
220
347
|
for (const part of parts) {
|
|
@@ -269,8 +396,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
|
|
|
269
396
|
});
|
|
270
397
|
}
|
|
271
398
|
function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
|
|
272
|
-
const
|
|
399
|
+
const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
|
|
273
400
|
return WAITING_SCENARIOS.map((extraPerVariant) => {
|
|
401
|
+
const random = makeDeterministicRandom(seed);
|
|
274
402
|
let regretSum = 0;
|
|
275
403
|
for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
|
|
276
404
|
const trueRateA = sampleBeta(alphaA, betaA, random);
|
|
@@ -296,12 +424,14 @@ function analyze({
|
|
|
296
424
|
minEffect = DEFAULT_MIN_EFFECT,
|
|
297
425
|
samples = DEFAULT_SAMPLES,
|
|
298
426
|
seed = null,
|
|
299
|
-
includeChart = false
|
|
427
|
+
includeChart = false,
|
|
428
|
+
prior = [PRIOR_ALPHA, PRIOR_BETA]
|
|
300
429
|
}) {
|
|
301
|
-
const
|
|
302
|
-
const
|
|
303
|
-
const
|
|
304
|
-
const
|
|
430
|
+
const [priorAlpha, priorBeta] = validatePrior(prior);
|
|
431
|
+
const alphaA = priorAlpha + conversionsA;
|
|
432
|
+
const betaA = priorBeta + (visitorsA - conversionsA);
|
|
433
|
+
const alphaB = priorAlpha + conversionsB;
|
|
434
|
+
const betaB = priorBeta + (visitorsB - conversionsB);
|
|
305
435
|
const rateA = conversionsA / visitorsA;
|
|
306
436
|
const rateB = conversionsB / visitorsB;
|
|
307
437
|
const posteriorMeanA = alphaA / (alphaA + betaA);
|
|
@@ -336,7 +466,7 @@ function analyze({
|
|
|
336
466
|
conversionsB,
|
|
337
467
|
threshold,
|
|
338
468
|
minEffect,
|
|
339
|
-
prior: [
|
|
469
|
+
prior: [priorAlpha, priorBeta],
|
|
340
470
|
samples,
|
|
341
471
|
seed: resolvedSeed
|
|
342
472
|
},
|
|
@@ -354,7 +484,9 @@ function analyze({
|
|
|
354
484
|
aBeatsB: 1 - pBBeatsA,
|
|
355
485
|
meaningfulLift: probabilityMeaningfulLift,
|
|
356
486
|
meaningfulHarm: probabilityMeaningfulHarm,
|
|
357
|
-
|
|
487
|
+
/* The exact complement of the closed-form win probability. Estimating it
|
|
488
|
+
from the samples left the two halves summing to slightly less than 1. */
|
|
489
|
+
harmful: 1 - pBBeatsA,
|
|
358
490
|
liftAbove1pct: share((lift) => lift > 0.01),
|
|
359
491
|
liftAbove5pct: share((lift) => lift > 0.05),
|
|
360
492
|
harmBeyond5pct: share((lift) => lift < -0.05),
|
|
@@ -366,7 +498,9 @@ function analyze({
|
|
|
366
498
|
absoluteDiff: posteriorMeanB - posteriorMeanA,
|
|
367
499
|
absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
|
|
368
500
|
observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
|
|
369
|
-
observedAbsoluteDiff: rateB - rateA
|
|
501
|
+
observedAbsoluteDiff: rateB - rateA,
|
|
502
|
+
relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
|
|
503
|
+
relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
|
|
370
504
|
},
|
|
371
505
|
expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
|
|
372
506
|
variants: {
|
|
@@ -380,12 +514,13 @@ function analyze({
|
|
|
380
514
|
},
|
|
381
515
|
waiting: waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow),
|
|
382
516
|
frequentist: { pValueTwoSided: twoSidedPValue(visitorsA, conversionsA, visitorsB, conversionsB) },
|
|
383
|
-
shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect })
|
|
517
|
+
shareUrl: shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] })
|
|
384
518
|
};
|
|
385
519
|
if (includeChart) {
|
|
386
520
|
const sortedA = [...samplesA].sort((left, right) => left - right);
|
|
387
521
|
const sortedB = [...samplesB].sort((left, right) => left - right);
|
|
388
522
|
result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
|
|
523
|
+
result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
|
|
389
524
|
}
|
|
390
525
|
return result;
|
|
391
526
|
}
|
|
@@ -467,9 +602,11 @@ function project({
|
|
|
467
602
|
stopRule = "lift",
|
|
468
603
|
maxLoss = DEFAULT_MAX_LOSS,
|
|
469
604
|
simulations = DEFAULT_SIMULATIONS,
|
|
470
|
-
seed = null
|
|
605
|
+
seed = null,
|
|
606
|
+
prior = [PRIOR_ALPHA, PRIOR_BETA]
|
|
471
607
|
}) {
|
|
472
|
-
const
|
|
608
|
+
const [priorAlpha, priorBeta] = validatePrior(prior);
|
|
609
|
+
const current = analyze({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
|
|
473
610
|
const resolvedSeed = seed ?? deriveSeed(visitorsA, conversionsA, visitorsB, conversionsB, dailyVisitors, horizonDays, ["lift", "superiority", "loss"].indexOf(stopRule), simulations);
|
|
474
611
|
const inputs = {
|
|
475
612
|
visitorsA,
|
|
@@ -478,7 +615,7 @@ function project({
|
|
|
478
615
|
conversionsB,
|
|
479
616
|
threshold,
|
|
480
617
|
minEffect,
|
|
481
|
-
prior: [
|
|
618
|
+
prior: [priorAlpha, priorBeta],
|
|
482
619
|
dailyVisitors,
|
|
483
620
|
horizonDays,
|
|
484
621
|
stopRule,
|
|
@@ -486,7 +623,7 @@ function project({
|
|
|
486
623
|
simulations,
|
|
487
624
|
seed: resolvedSeed
|
|
488
625
|
};
|
|
489
|
-
const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect });
|
|
626
|
+
const link = shareUrl({ visitorsA, conversionsA, visitorsB, conversionsB, threshold, minEffect, prior: [priorAlpha, priorBeta] });
|
|
490
627
|
const diagnostic = stopRule === "lift" && liftNearThreshold(current, minEffect) ? "lift_near_threshold" : null;
|
|
491
628
|
const decisionNow = decisionNowFrom(current, stopRule, threshold, maxLoss);
|
|
492
629
|
if (decisionNow) {
|
|
@@ -503,10 +640,10 @@ function project({
|
|
|
503
640
|
shareUrl: link
|
|
504
641
|
};
|
|
505
642
|
}
|
|
506
|
-
const alphaA0 =
|
|
507
|
-
const betaA0 =
|
|
508
|
-
const alphaB0 =
|
|
509
|
-
const betaB0 =
|
|
643
|
+
const alphaA0 = priorAlpha + conversionsA;
|
|
644
|
+
const betaA0 = priorBeta + (visitorsA - conversionsA);
|
|
645
|
+
const alphaB0 = priorAlpha + conversionsB;
|
|
646
|
+
const betaB0 = priorBeta + (visitorsB - conversionsB);
|
|
510
647
|
const random = makeDeterministicRandom(resolvedSeed);
|
|
511
648
|
const decidedByDay = new Array(horizonDays + 1).fill(0);
|
|
512
649
|
const decisionDays = [];
|
|
@@ -562,6 +699,30 @@ function project({
|
|
|
562
699
|
shareUrl: link
|
|
563
700
|
};
|
|
564
701
|
}
|
|
702
|
+
|
|
703
|
+
// src/core/loss.ts
|
|
704
|
+
function integratedExpectedLoss(alphaA, betaA, alphaB, betaB) {
|
|
705
|
+
const GRID = 600;
|
|
706
|
+
const dx = 1 / GRID;
|
|
707
|
+
let cdfA = 0;
|
|
708
|
+
let cdfB = 0;
|
|
709
|
+
let weightedSumA = 0;
|
|
710
|
+
let weightedSumB = 0;
|
|
711
|
+
let shipB = 0;
|
|
712
|
+
let shipA = 0;
|
|
713
|
+
for (let i = 0; i <= GRID; i += 1) {
|
|
714
|
+
const x = clamp(i * dx, 1e-6, 1 - 1e-6);
|
|
715
|
+
const fA = betaPdf(x, alphaA, betaA);
|
|
716
|
+
const fB = betaPdf(x, alphaB, betaB);
|
|
717
|
+
shipB += fA * (x * cdfB - weightedSumB) * dx;
|
|
718
|
+
shipA += fB * (x * cdfA - weightedSumA) * dx;
|
|
719
|
+
cdfA += fA * dx;
|
|
720
|
+
weightedSumA += x * fA * dx;
|
|
721
|
+
cdfB += fB * dx;
|
|
722
|
+
weightedSumB += x * fB * dx;
|
|
723
|
+
}
|
|
724
|
+
return { shipA: Math.max(shipA, 0), shipB: Math.max(shipB, 0) };
|
|
725
|
+
}
|
|
565
726
|
export {
|
|
566
727
|
DECISION_LABELS,
|
|
567
728
|
DEFAULT_HORIZON_DAYS,
|
|
@@ -570,11 +731,21 @@ export {
|
|
|
570
731
|
DEFAULT_SAMPLES,
|
|
571
732
|
DEFAULT_SIMULATIONS,
|
|
572
733
|
DEFAULT_THRESHOLD,
|
|
734
|
+
PRIOR_ALPHA,
|
|
735
|
+
PRIOR_BETA,
|
|
573
736
|
WEB_TOOL_URL,
|
|
574
737
|
analyze,
|
|
738
|
+
betaPdf,
|
|
739
|
+
integratedExpectedLoss,
|
|
740
|
+
isDefaultPrior,
|
|
741
|
+
liftDensity,
|
|
742
|
+
logBeta,
|
|
743
|
+
logGamma,
|
|
575
744
|
plan,
|
|
576
745
|
probabilityBBeatsA,
|
|
577
746
|
project,
|
|
747
|
+
regularizedIncompleteBeta,
|
|
578
748
|
requiredSamplePerVariant,
|
|
579
|
-
shareUrl
|
|
749
|
+
shareUrl,
|
|
750
|
+
validatePrior
|
|
580
751
|
};
|
package/package.json
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bayes-ab",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Bayesian A/B test analysis on the command line, for humans and agents",
|
|
5
|
-
"keywords": [
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ab-testing",
|
|
7
|
+
"bayesian",
|
|
8
|
+
"statistics",
|
|
9
|
+
"cli",
|
|
10
|
+
"experimentation",
|
|
11
|
+
"conversion-rate"
|
|
12
|
+
],
|
|
6
13
|
"homepage": "https://www.joebrundage.com/tools/bayesian-ab-test",
|
|
7
14
|
"repository": {
|
|
8
15
|
"type": "git",
|
|
@@ -23,7 +30,11 @@
|
|
|
23
30
|
"import": "./dist/index.js"
|
|
24
31
|
}
|
|
25
32
|
},
|
|
26
|
-
"files": [
|
|
33
|
+
"files": [
|
|
34
|
+
"dist",
|
|
35
|
+
"README.md",
|
|
36
|
+
"LICENSE"
|
|
37
|
+
],
|
|
27
38
|
"engines": {
|
|
28
39
|
"node": ">=18.11"
|
|
29
40
|
},
|