bayes-ab 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -7
- package/dist/cli.js +66 -9
- package/dist/core/analyze.d.ts +4 -0
- package/dist/core/lift.d.ts +15 -0
- package/dist/core/loss.d.ts +13 -0
- package/dist/index.d.ts +5 -2
- package/dist/index.js +85 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ Bayes action now Ship B (expected regret 0.0050 pp, about 0.50 missed conve
|
|
|
23
23
|
P(B beats A) 95.91%
|
|
24
24
|
P(relative lift > +10%) 46.42%
|
|
25
25
|
P(relative lift < -10%) 0.02%
|
|
26
|
-
P(B is harmful at all) 4.
|
|
26
|
+
P(B is harmful at all) 4.09%
|
|
27
27
|
|
|
28
28
|
Relative lift +9.54% 95% CI -1.17% to +21.30%
|
|
29
29
|
Absolute difference +0.53 pp 95% CI -0.07 pp to +1.11 pp
|
|
@@ -173,16 +173,16 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --con
|
|
|
173
173
|
```json
|
|
174
174
|
{
|
|
175
175
|
"tool": "bayes-ab",
|
|
176
|
-
"version": "0.
|
|
176
|
+
"version": "0.2.0",
|
|
177
177
|
"command": "analyze",
|
|
178
|
-
"inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed":
|
|
178
|
+
"inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 116218 },
|
|
179
179
|
"decision": { "status": "continue", "label": "Continue test", "evidence": "low", "maturity": "building", "maturityProgress": 0.462241, "bayesActionNow": "B", "expectedRegretNow": 0.00005 },
|
|
180
|
-
"probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.
|
|
181
|
-
"effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253 },
|
|
180
|
+
"probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040886, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
|
|
181
|
+
"effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253, "relativeLiftMedian": 0.095267, "relativeLiftIQR": [0.056954, 0.134715] },
|
|
182
182
|
"expectedLoss": { "shipA": 0.00528, "shipB": 0.00005 },
|
|
183
183
|
"variants": { "A": { "...": "..." }, "B": { "...": "..." } },
|
|
184
184
|
"sampleSize": { "requiredPerVariant": 25636, "additionalPerVariant": 13786, "detectableEffectNow": 0.147083 },
|
|
185
|
-
"waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.
|
|
185
|
+
"waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.000048, "valueOfWaiting": 0.000002 }, "..." ],
|
|
186
186
|
"frequentist": { "pValueTwoSided": 0.081709 },
|
|
187
187
|
"shareUrl": "https://www.joebrundage.com/tools/bayesian-ab-test?visitorsA=12000&conversionsA=660&visitorsB=11850&conversionsB=714&thresholdPercent=95&meaningfulLiftPercent=10"
|
|
188
188
|
}
|
|
@@ -229,10 +229,23 @@ const result = analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850,
|
|
|
229
229
|
result.probability.bBeatsA // 0.9591138673472563
|
|
230
230
|
```
|
|
231
231
|
|
|
232
|
+
`analyze({ ..., includeChart: true })` adds `chart` (both posterior densities) and `liftChart` (the exact density of the relative lift, peak scaled to 1), which is what the web tool plots.
|
|
233
|
+
|
|
234
|
+
The primitives behind it are exported too, for anything that needs the same math without a full analysis:
|
|
235
|
+
|
|
236
|
+
| Export | What it does |
|
|
237
|
+
|---|---|
|
|
238
|
+
| `betaPdf(x, alpha, beta)`, `logBeta`, `logGamma` | Beta density and its log-gamma building blocks |
|
|
239
|
+
| `probabilityBBeatsA(alphaA, betaA, alphaB, betaB)` | Exact P(B > A) for two Beta posteriors |
|
|
240
|
+
| `integratedExpectedLoss(alphaA, betaA, alphaB, betaB)` | Expected loss of shipping each variant by numerical integration, no sampling |
|
|
241
|
+
| `liftDensity(alphaA, betaA, alphaB, betaB, sortedLifts)` | Exact relative-lift density over the range the sorted lifts span |
|
|
242
|
+
|
|
243
|
+
This package is the source of truth for the math on joebrundage.com: the [calculator](https://www.joebrundage.com/tools/bayesian-ab-test) and the [primer](https://www.joebrundage.com/tools/bayesian-ab-testing-primer) both import it. Change the math here, publish, and bump the site's dependency.
|
|
244
|
+
|
|
232
245
|
## Method
|
|
233
246
|
|
|
234
247
|
- Each conversion rate gets a Beta(1,1) prior. Posteriors are Beta(1 + conversions, 1 + non-conversions).
|
|
235
|
-
- P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html).
|
|
248
|
+
- P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html). P(B is harmful) is its exact complement.
|
|
236
249
|
- Lift intervals, threshold probabilities and expected loss come from 12,000 seeded Monte Carlo draws.
|
|
237
250
|
- The meaningful effect is relative and symmetric: `--min-effect 10` means B must be 10% better than A to count as a win or 10% worse to count as a loss.
|
|
238
251
|
- `project` samples true rates from the exact posterior, then uses a normal approximation to the posterior when re-checking the stopping rule on each simulated day. Day zero is checked exactly.
|
package/dist/cli.js
CHANGED
|
@@ -158,6 +158,50 @@ function round(value, decimals = 6) {
|
|
|
158
158
|
return Math.round(value * factor) / factor;
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
+
// src/core/lift.ts
|
|
162
|
+
var LIFT_DENSITY_POINTS = 160;
|
|
163
|
+
var LIFT_INTEGRATION_STEPS = 400;
|
|
164
|
+
function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
|
|
165
|
+
const lower = quantile(sortedRelativeLifts, 2e-3);
|
|
166
|
+
const upper = quantile(sortedRelativeLifts, 0.998);
|
|
167
|
+
const span = upper - lower;
|
|
168
|
+
if (!Number.isFinite(span) || span <= 0) {
|
|
169
|
+
return [
|
|
170
|
+
{ relativeLift: lower, density: 0 },
|
|
171
|
+
{ relativeLift: lower, density: 1 },
|
|
172
|
+
{ relativeLift: lower, density: 0 }
|
|
173
|
+
];
|
|
174
|
+
}
|
|
175
|
+
const totalA = alphaA + betaA;
|
|
176
|
+
const meanA = alphaA / totalA;
|
|
177
|
+
const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
|
|
178
|
+
const aLow = Math.max(1e-9, meanA - 10 * sdA);
|
|
179
|
+
const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
|
|
180
|
+
const lifts = Array.from(
|
|
181
|
+
{ length: LIFT_DENSITY_POINTS },
|
|
182
|
+
(_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
|
|
183
|
+
);
|
|
184
|
+
const densities = lifts.map((relativeLift) => {
|
|
185
|
+
const ratio = 1 + relativeLift;
|
|
186
|
+
if (ratio <= 0) return 0;
|
|
187
|
+
const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
|
|
188
|
+
if (aHigh <= aLow) return 0;
|
|
189
|
+
const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
|
|
190
|
+
let total = 0;
|
|
191
|
+
for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
|
|
192
|
+
const a = aLow + step * index;
|
|
193
|
+
const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
|
|
194
|
+
total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
|
|
195
|
+
}
|
|
196
|
+
return total * step / 3;
|
|
197
|
+
});
|
|
198
|
+
const peak = Math.max(...densities, Number.MIN_VALUE);
|
|
199
|
+
return lifts.map((relativeLift, index) => ({
|
|
200
|
+
relativeLift,
|
|
201
|
+
density: densities[index] / peak
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
|
|
161
205
|
// src/core/plan.ts
|
|
162
206
|
var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
|
|
163
207
|
function requiredSamplePerVariant({
|
|
@@ -215,7 +259,7 @@ var PRIOR_ALPHA = 1;
|
|
|
215
259
|
var PRIOR_BETA = 1;
|
|
216
260
|
var POSTERIOR_CHART_POINTS = 96;
|
|
217
261
|
var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
|
|
218
|
-
var WAITING_VALUE_SIMULATIONS =
|
|
262
|
+
var WAITING_VALUE_SIMULATIONS = 6e3;
|
|
219
263
|
var DECISION_LABELS = {
|
|
220
264
|
ship_b: "Ship Variant B",
|
|
221
265
|
keep_a: "Keep Variant A",
|
|
@@ -276,8 +320,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
|
|
|
276
320
|
});
|
|
277
321
|
}
|
|
278
322
|
function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
|
|
279
|
-
const
|
|
323
|
+
const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
|
|
280
324
|
return WAITING_SCENARIOS.map((extraPerVariant) => {
|
|
325
|
+
const random = makeDeterministicRandom(seed);
|
|
281
326
|
let regretSum = 0;
|
|
282
327
|
for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
|
|
283
328
|
const trueRateA = sampleBeta(alphaA, betaA, random);
|
|
@@ -361,7 +406,9 @@ function analyze({
|
|
|
361
406
|
aBeatsB: 1 - pBBeatsA,
|
|
362
407
|
meaningfulLift: probabilityMeaningfulLift,
|
|
363
408
|
meaningfulHarm: probabilityMeaningfulHarm,
|
|
364
|
-
|
|
409
|
+
/* The exact complement of the closed-form win probability. Estimating it
|
|
410
|
+
from the samples left the two halves summing to slightly less than 1. */
|
|
411
|
+
harmful: 1 - pBBeatsA,
|
|
365
412
|
liftAbove1pct: share((lift) => lift > 0.01),
|
|
366
413
|
liftAbove5pct: share((lift) => lift > 0.05),
|
|
367
414
|
harmBeyond5pct: share((lift) => lift < -0.05),
|
|
@@ -373,7 +420,9 @@ function analyze({
|
|
|
373
420
|
absoluteDiff: posteriorMeanB - posteriorMeanA,
|
|
374
421
|
absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
|
|
375
422
|
observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
|
|
376
|
-
observedAbsoluteDiff: rateB - rateA
|
|
423
|
+
observedAbsoluteDiff: rateB - rateA,
|
|
424
|
+
relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
|
|
425
|
+
relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
|
|
377
426
|
},
|
|
378
427
|
expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
|
|
379
428
|
variants: {
|
|
@@ -393,6 +442,7 @@ function analyze({
|
|
|
393
442
|
const sortedA = [...samplesA].sort((left, right) => left - right);
|
|
394
443
|
const sortedB = [...samplesB].sort((left, right) => left - right);
|
|
395
444
|
result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
|
|
445
|
+
result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
|
|
396
446
|
}
|
|
397
447
|
return result;
|
|
398
448
|
}
|
|
@@ -819,7 +869,7 @@ OUTPUT
|
|
|
819
869
|
so piping or calling from an agent gets JSON automatically.
|
|
820
870
|
--pretty Indent the JSON.
|
|
821
871
|
--brief One-line human summary (analyze and project).
|
|
822
|
-
--include-chart Add
|
|
872
|
+
--include-chart Add posterior and lift density curves to analyze's JSON.
|
|
823
873
|
--samples N Monte Carlo samples for intervals and loss. default 12000
|
|
824
874
|
--seed N RNG seed. Defaults to a value derived from the inputs, so identical
|
|
825
875
|
inputs always give identical output.
|
|
@@ -1217,7 +1267,7 @@ var ANALYZE_SCHEMA = {
|
|
|
1217
1267
|
aBeatsB: fraction,
|
|
1218
1268
|
meaningfulLift: { ...fraction, description: "P(relative lift > minEffect)" },
|
|
1219
1269
|
meaningfulHarm: { ...fraction, description: "P(relative lift < -minEffect)" },
|
|
1220
|
-
harmful: { ...fraction, description: "P(
|
|
1270
|
+
harmful: { ...fraction, description: "P(B < A), the exact complement of bBeatsA" },
|
|
1221
1271
|
liftAbove1pct: fraction,
|
|
1222
1272
|
liftAbove5pct: fraction,
|
|
1223
1273
|
harmBeyond5pct: fraction,
|
|
@@ -1229,7 +1279,9 @@ var ANALYZE_SCHEMA = {
|
|
|
1229
1279
|
absoluteDiff: number,
|
|
1230
1280
|
absoluteDiffCI95: pair(number),
|
|
1231
1281
|
observedRelativeLift: { type: ["number", "null"] },
|
|
1232
|
-
observedAbsoluteDiff: number
|
|
1282
|
+
observedAbsoluteDiff: number,
|
|
1283
|
+
relativeLiftMedian: number,
|
|
1284
|
+
relativeLiftIQR: pair(number)
|
|
1233
1285
|
}),
|
|
1234
1286
|
expectedLoss: object({ shipA: fraction, shipB: fraction }),
|
|
1235
1287
|
variants: object({
|
|
@@ -1251,6 +1303,11 @@ var ANALYZE_SCHEMA = {
|
|
|
1251
1303
|
description: "Only with --include-chart.",
|
|
1252
1304
|
items: object({ rate: fraction, densityA: number, densityB: number })
|
|
1253
1305
|
},
|
|
1306
|
+
liftChart: {
|
|
1307
|
+
type: "array",
|
|
1308
|
+
description: "Only with --include-chart. Exact density of the relative lift, peak scaled to 1.",
|
|
1309
|
+
items: object({ relativeLift: number, density: number })
|
|
1310
|
+
},
|
|
1254
1311
|
shareUrl: { type: "string", format: "uri" }
|
|
1255
1312
|
}, [
|
|
1256
1313
|
"tool",
|
|
@@ -1343,7 +1400,7 @@ var SCHEMAS = {
|
|
|
1343
1400
|
};
|
|
1344
1401
|
|
|
1345
1402
|
// src/cli/main.ts
|
|
1346
|
-
var VERSION = true ? "0.
|
|
1403
|
+
var VERSION = true ? "0.2.0" : "dev";
|
|
1347
1404
|
var JSON_DECIMALS = 6;
|
|
1348
1405
|
function roundDeep(value) {
|
|
1349
1406
|
if (typeof value === "number") return round(value, JSON_DECIMALS);
|
|
@@ -1367,7 +1424,7 @@ function envelope2(command, result, derivedFrom2) {
|
|
|
1367
1424
|
return { tool: "bayes-ab", version: VERSION, command, ...result, inputs };
|
|
1368
1425
|
}
|
|
1369
1426
|
function stripChart(result) {
|
|
1370
|
-
const { chart, ...rest } = result;
|
|
1427
|
+
const { chart, liftChart, ...rest } = result;
|
|
1371
1428
|
return rest;
|
|
1372
1429
|
}
|
|
1373
1430
|
function run(argv, io) {
|
package/dist/core/analyze.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type LiftDensityPoint } from "./lift.ts";
|
|
1
2
|
export declare const DEFAULT_THRESHOLD = 0.95;
|
|
2
3
|
export declare const DEFAULT_MIN_EFFECT = 0.1;
|
|
3
4
|
export declare const DEFAULT_SAMPLES = 12000;
|
|
@@ -75,6 +76,8 @@ export type AnalyzeResult = {
|
|
|
75
76
|
absoluteDiffCI95: [number, number];
|
|
76
77
|
observedRelativeLift: number | null;
|
|
77
78
|
observedAbsoluteDiff: number;
|
|
79
|
+
relativeLiftMedian: number;
|
|
80
|
+
relativeLiftIQR: [number, number];
|
|
78
81
|
};
|
|
79
82
|
expectedLoss: {
|
|
80
83
|
shipA: number;
|
|
@@ -94,6 +97,7 @@ export type AnalyzeResult = {
|
|
|
94
97
|
pValueTwoSided: number;
|
|
95
98
|
};
|
|
96
99
|
chart?: ChartPoint[];
|
|
100
|
+
liftChart?: LiftDensityPoint[];
|
|
97
101
|
shareUrl: string;
|
|
98
102
|
};
|
|
99
103
|
export declare function deriveSeed(...parts: number[]): number;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type LiftDensityPoint = {
|
|
2
|
+
relativeLift: number;
|
|
3
|
+
/** Scaled so the peak is 1. */
|
|
4
|
+
density: number;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Density of B's relative lift over A, computed exactly rather than
|
|
8
|
+
* histogrammed from the Monte Carlo samples. Writing B = A(1 + r), the ratio
|
|
9
|
+
* density is f(r) = ∫ a · f_A(a) · f_B(a(1+r)) da, which is one Simpson
|
|
10
|
+
* integral per grid point. A histogram of 12k draws carries ~10% sampling
|
|
11
|
+
* noise per bin, and on a wide posterior that reads as a jagged curve.
|
|
12
|
+
*
|
|
13
|
+
* The sorted Monte Carlo lifts only set the plotted range (0.2% to 99.8%).
|
|
14
|
+
*/
|
|
15
|
+
export declare function liftDensity(alphaA: number, betaA: number, alphaB: number, betaB: number, sortedRelativeLifts: number[]): LiftDensityPoint[];
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Expected loss of each choice by direct numerical integration over a 600-step
|
|
3
|
+
* grid, with no sampling. Deterministic and smooth in its inputs, so it suits
|
|
4
|
+
* curves that sweep sample size; `analyze` uses Monte Carlo instead because it
|
|
5
|
+
* shares those draws with the lift interval and threshold probabilities.
|
|
6
|
+
*
|
|
7
|
+
* Losses are absolute conversion-rate differences (0.001 = 0.1 pp):
|
|
8
|
+
* shipA = E[max(B − A, 0)], shipB = E[max(A − B, 0)].
|
|
9
|
+
*/
|
|
10
|
+
export declare function integratedExpectedLoss(alphaA: number, betaA: number, alphaB: number, betaB: number): {
|
|
11
|
+
shipA: number;
|
|
12
|
+
shipB: number;
|
|
13
|
+
};
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD } from "./core/analyze.ts";
|
|
1
|
+
export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD, PRIOR_ALPHA, PRIOR_BETA } from "./core/analyze.ts";
|
|
2
2
|
export type { AnalyzeInputs, AnalyzeResult, ChartPoint, DecisionStatus, EvidenceStrength, Maturity, Variant, VariantSummary, WaitingScenario, } from "./core/analyze.ts";
|
|
3
3
|
export { project, DEFAULT_HORIZON_DAYS, DEFAULT_MAX_LOSS, DEFAULT_SIMULATIONS } from "./core/project.ts";
|
|
4
4
|
export type { ProjectInputs, ProjectResult, ProjectedDecision, StopRule } from "./core/project.ts";
|
|
5
5
|
export { plan, requiredSamplePerVariant } from "./core/plan.ts";
|
|
6
6
|
export type { PlanInputs, PlanResult, PlanRow } from "./core/plan.ts";
|
|
7
7
|
export { shareUrl, WEB_TOOL_URL } from "./core/share.ts";
|
|
8
|
-
export { probabilityBBeatsA } from "./core/math.ts";
|
|
8
|
+
export { betaPdf, logBeta, logGamma, probabilityBBeatsA } from "./core/math.ts";
|
|
9
|
+
export { liftDensity } from "./core/lift.ts";
|
|
10
|
+
export type { LiftDensityPoint } from "./core/lift.ts";
|
|
11
|
+
export { integratedExpectedLoss } from "./core/loss.ts";
|
package/dist/index.js
CHANGED
|
@@ -151,6 +151,50 @@ function normalPdf(x) {
|
|
|
151
151
|
return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
+
// src/core/lift.ts
|
|
155
|
+
var LIFT_DENSITY_POINTS = 160;
|
|
156
|
+
var LIFT_INTEGRATION_STEPS = 400;
|
|
157
|
+
function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
|
|
158
|
+
const lower = quantile(sortedRelativeLifts, 2e-3);
|
|
159
|
+
const upper = quantile(sortedRelativeLifts, 0.998);
|
|
160
|
+
const span = upper - lower;
|
|
161
|
+
if (!Number.isFinite(span) || span <= 0) {
|
|
162
|
+
return [
|
|
163
|
+
{ relativeLift: lower, density: 0 },
|
|
164
|
+
{ relativeLift: lower, density: 1 },
|
|
165
|
+
{ relativeLift: lower, density: 0 }
|
|
166
|
+
];
|
|
167
|
+
}
|
|
168
|
+
const totalA = alphaA + betaA;
|
|
169
|
+
const meanA = alphaA / totalA;
|
|
170
|
+
const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
|
|
171
|
+
const aLow = Math.max(1e-9, meanA - 10 * sdA);
|
|
172
|
+
const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
|
|
173
|
+
const lifts = Array.from(
|
|
174
|
+
{ length: LIFT_DENSITY_POINTS },
|
|
175
|
+
(_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
|
|
176
|
+
);
|
|
177
|
+
const densities = lifts.map((relativeLift) => {
|
|
178
|
+
const ratio = 1 + relativeLift;
|
|
179
|
+
if (ratio <= 0) return 0;
|
|
180
|
+
const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
|
|
181
|
+
if (aHigh <= aLow) return 0;
|
|
182
|
+
const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
|
|
183
|
+
let total = 0;
|
|
184
|
+
for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
|
|
185
|
+
const a = aLow + step * index;
|
|
186
|
+
const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
|
|
187
|
+
total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
|
|
188
|
+
}
|
|
189
|
+
return total * step / 3;
|
|
190
|
+
});
|
|
191
|
+
const peak = Math.max(...densities, Number.MIN_VALUE);
|
|
192
|
+
return lifts.map((relativeLift, index) => ({
|
|
193
|
+
relativeLift,
|
|
194
|
+
density: densities[index] / peak
|
|
195
|
+
}));
|
|
196
|
+
}
|
|
197
|
+
|
|
154
198
|
// src/core/plan.ts
|
|
155
199
|
var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
|
|
156
200
|
function requiredSamplePerVariant({
|
|
@@ -208,7 +252,7 @@ var PRIOR_ALPHA = 1;
|
|
|
208
252
|
var PRIOR_BETA = 1;
|
|
209
253
|
var POSTERIOR_CHART_POINTS = 96;
|
|
210
254
|
var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
|
|
211
|
-
var WAITING_VALUE_SIMULATIONS =
|
|
255
|
+
var WAITING_VALUE_SIMULATIONS = 6e3;
|
|
212
256
|
var DECISION_LABELS = {
|
|
213
257
|
ship_b: "Ship Variant B",
|
|
214
258
|
keep_a: "Keep Variant A",
|
|
@@ -269,8 +313,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
|
|
|
269
313
|
});
|
|
270
314
|
}
|
|
271
315
|
function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
|
|
272
|
-
const
|
|
316
|
+
const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
|
|
273
317
|
return WAITING_SCENARIOS.map((extraPerVariant) => {
|
|
318
|
+
const random = makeDeterministicRandom(seed);
|
|
274
319
|
let regretSum = 0;
|
|
275
320
|
for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
|
|
276
321
|
const trueRateA = sampleBeta(alphaA, betaA, random);
|
|
@@ -354,7 +399,9 @@ function analyze({
|
|
|
354
399
|
aBeatsB: 1 - pBBeatsA,
|
|
355
400
|
meaningfulLift: probabilityMeaningfulLift,
|
|
356
401
|
meaningfulHarm: probabilityMeaningfulHarm,
|
|
357
|
-
|
|
402
|
+
/* The exact complement of the closed-form win probability. Estimating it
|
|
403
|
+
from the samples left the two halves summing to slightly less than 1. */
|
|
404
|
+
harmful: 1 - pBBeatsA,
|
|
358
405
|
liftAbove1pct: share((lift) => lift > 0.01),
|
|
359
406
|
liftAbove5pct: share((lift) => lift > 0.05),
|
|
360
407
|
harmBeyond5pct: share((lift) => lift < -0.05),
|
|
@@ -366,7 +413,9 @@ function analyze({
|
|
|
366
413
|
absoluteDiff: posteriorMeanB - posteriorMeanA,
|
|
367
414
|
absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
|
|
368
415
|
observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
|
|
369
|
-
observedAbsoluteDiff: rateB - rateA
|
|
416
|
+
observedAbsoluteDiff: rateB - rateA,
|
|
417
|
+
relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
|
|
418
|
+
relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
|
|
370
419
|
},
|
|
371
420
|
expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
|
|
372
421
|
variants: {
|
|
@@ -386,6 +435,7 @@ function analyze({
|
|
|
386
435
|
const sortedA = [...samplesA].sort((left, right) => left - right);
|
|
387
436
|
const sortedB = [...samplesB].sort((left, right) => left - right);
|
|
388
437
|
result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
|
|
438
|
+
result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
|
|
389
439
|
}
|
|
390
440
|
return result;
|
|
391
441
|
}
|
|
@@ -562,6 +612,30 @@ function project({
|
|
|
562
612
|
shareUrl: link
|
|
563
613
|
};
|
|
564
614
|
}
|
|
615
|
+
|
|
616
|
+
// src/core/loss.ts
|
|
617
|
+
function integratedExpectedLoss(alphaA, betaA, alphaB, betaB) {
|
|
618
|
+
const GRID = 600;
|
|
619
|
+
const dx = 1 / GRID;
|
|
620
|
+
let cdfA = 0;
|
|
621
|
+
let cdfB = 0;
|
|
622
|
+
let weightedSumA = 0;
|
|
623
|
+
let weightedSumB = 0;
|
|
624
|
+
let shipB = 0;
|
|
625
|
+
let shipA = 0;
|
|
626
|
+
for (let i = 0; i <= GRID; i += 1) {
|
|
627
|
+
const x = clamp(i * dx, 1e-6, 1 - 1e-6);
|
|
628
|
+
const fA = betaPdf(x, alphaA, betaA);
|
|
629
|
+
const fB = betaPdf(x, alphaB, betaB);
|
|
630
|
+
shipB += fA * (x * cdfB - weightedSumB) * dx;
|
|
631
|
+
shipA += fB * (x * cdfA - weightedSumA) * dx;
|
|
632
|
+
cdfA += fA * dx;
|
|
633
|
+
weightedSumA += x * fA * dx;
|
|
634
|
+
cdfB += fB * dx;
|
|
635
|
+
weightedSumB += x * fB * dx;
|
|
636
|
+
}
|
|
637
|
+
return { shipA: Math.max(shipA, 0), shipB: Math.max(shipB, 0) };
|
|
638
|
+
}
|
|
565
639
|
export {
|
|
566
640
|
DECISION_LABELS,
|
|
567
641
|
DEFAULT_HORIZON_DAYS,
|
|
@@ -570,8 +644,15 @@ export {
|
|
|
570
644
|
DEFAULT_SAMPLES,
|
|
571
645
|
DEFAULT_SIMULATIONS,
|
|
572
646
|
DEFAULT_THRESHOLD,
|
|
647
|
+
PRIOR_ALPHA,
|
|
648
|
+
PRIOR_BETA,
|
|
573
649
|
WEB_TOOL_URL,
|
|
574
650
|
analyze,
|
|
651
|
+
betaPdf,
|
|
652
|
+
integratedExpectedLoss,
|
|
653
|
+
liftDensity,
|
|
654
|
+
logBeta,
|
|
655
|
+
logGamma,
|
|
575
656
|
plan,
|
|
576
657
|
probabilityBBeatsA,
|
|
577
658
|
project,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bayes-ab",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Bayesian A/B test analysis on the command line, for humans and agents",
|
|
5
5
|
"keywords": ["ab-testing", "bayesian", "statistics", "cli", "experimentation", "conversion-rate"],
|
|
6
6
|
"homepage": "https://www.joebrundage.com/tools/bayesian-ab-test",
|