bayes-ab 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -23,7 +23,7 @@ Bayes action now Ship B (expected regret 0.0050 pp, about 0.50 missed conve
23
23
  P(B beats A) 95.91%
24
24
  P(relative lift > +10%) 46.42%
25
25
  P(relative lift < -10%) 0.02%
26
- P(B is harmful at all) 4.06%
26
+ P(B is harmful at all) 4.09%
27
27
 
28
28
  Relative lift +9.54% 95% CI -1.17% to +21.30%
29
29
  Absolute difference +0.53 pp 95% CI -0.07 pp to +1.11 pp
@@ -173,16 +173,16 @@ bayes-ab analyze --visitors-a 12000 --conversions-a 660 --visitors-b 11850 --con
173
173
  ```json
174
174
  {
175
175
  "tool": "bayes-ab",
176
- "version": "0.1.0",
176
+ "version": "0.2.0",
177
177
  "command": "analyze",
178
- "inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 112396 },
178
+ "inputs": { "visitorsA": 12000, "conversionsA": 660, "visitorsB": 11850, "conversionsB": 714, "threshold": 0.95, "minEffect": 0.1, "prior": [1, 1], "samples": 12000, "seed": 116218 },
179
179
  "decision": { "status": "continue", "label": "Continue test", "evidence": "low", "maturity": "building", "maturityProgress": 0.462241, "bayesActionNow": "B", "expectedRegretNow": 0.00005 },
180
- "probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040583, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
181
- "effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253 },
180
+ "probability": { "bBeatsA": 0.959114, "aBeatsB": 0.040886, "meaningfulLift": 0.464167, "meaningfulHarm": 0.000167, "harmful": 0.040886, "liftAbove1pct": 0.937833, "liftAbove5pct": 0.788417, "harmBeyond5pct": 0.00325, "harmBeyond10pct": 0.000167 },
181
+ "effect": { "relativeLift": 0.095384, "relativeLiftCI95": [-0.011657, 0.213021], "absoluteDiff": 0.005253, "absoluteDiffCI95": [-0.000675, 0.011123], "observedRelativeLift": 0.095512, "observedAbsoluteDiff": 0.005253, "relativeLiftMedian": 0.095267, "relativeLiftIQR": [0.056954, 0.134715] },
182
182
  "expectedLoss": { "shipA": 0.00528, "shipB": 0.00005 },
183
183
  "variants": { "A": { "...": "..." }, "B": { "...": "..." } },
184
184
  "sampleSize": { "requiredPerVariant": 25636, "additionalPerVariant": 13786, "detectableEffectNow": 0.147083 },
185
- "waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.00006, "valueOfWaiting": 0 }, "..." ],
185
+ "waiting": [ { "extraPerVariant": 1000, "expectedRegret": 0.000048, "valueOfWaiting": 0.000002 }, "..." ],
186
186
  "frequentist": { "pValueTwoSided": 0.081709 },
187
187
  "shareUrl": "https://www.joebrundage.com/tools/bayesian-ab-test?visitorsA=12000&conversionsA=660&visitorsB=11850&conversionsB=714&thresholdPercent=95&meaningfulLiftPercent=10"
188
188
  }
@@ -229,10 +229,23 @@ const result = analyze({ visitorsA: 12000, conversionsA: 660, visitorsB: 11850,
229
229
  result.probability.bBeatsA // 0.9591138673472563
230
230
  ```
231
231
 
232
+ `analyze({ ..., includeChart: true })` adds `chart` (both posterior densities) and `liftChart` (the exact density of the relative lift, peak scaled to 1), which is what the web tool plots.
233
+
234
+ The primitives behind it are exported too, for anything that needs the same math without a full analysis:
235
+
236
+ | Export | What it does |
237
+ |---|---|
238
+ | `betaPdf(x, alpha, beta)`, `logBeta`, `logGamma` | Beta density and its log-gamma building blocks |
239
+ | `probabilityBBeatsA(alphaA, betaA, alphaB, betaB)` | Exact P(B > A) for two Beta posteriors |
240
+ | `integratedExpectedLoss(alphaA, betaA, alphaB, betaB)` | Expected loss of shipping each variant by numerical integration, no sampling |
241
+ | `liftDensity(alphaA, betaA, alphaB, betaB, sortedLifts)` | Exact relative-lift density over the range the sorted lifts span |
242
+
243
+ This package is the source of truth for the math on joebrundage.com: the [calculator](https://www.joebrundage.com/tools/bayesian-ab-test) and the [primer](https://www.joebrundage.com/tools/bayesian-ab-testing-primer) both import it. Change the math here, publish, and bump the site's dependency.
244
+
232
245
  ## Method
233
246
 
234
247
  - Each conversion rate gets a Beta(1,1) prior. Posteriors are Beta(1 + conversions, 1 + non-conversions).
235
- - P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html).
248
+ - P(B beats A) is the exact closed form from [Evan Miller](https://www.evanmiller.org/bayesian-ab-testing.html). P(B is harmful) is its exact complement.
236
249
  - Lift intervals, threshold probabilities and expected loss come from 12,000 seeded Monte Carlo draws.
237
250
  - The meaningful effect is relative and symmetric: `--min-effect 10` means B must be 10% better than A to count as a win or 10% worse to count as a loss.
238
251
  - `project` samples true rates from the exact posterior, then uses a normal approximation to the posterior when re-checking the stopping rule on each simulated day. Day zero is checked exactly.
package/dist/cli.js CHANGED
@@ -158,6 +158,50 @@ function round(value, decimals = 6) {
158
158
  return Math.round(value * factor) / factor;
159
159
  }
160
160
 
161
+ // src/core/lift.ts
162
+ var LIFT_DENSITY_POINTS = 160;
163
+ var LIFT_INTEGRATION_STEPS = 400;
164
+ function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
165
+ const lower = quantile(sortedRelativeLifts, 2e-3);
166
+ const upper = quantile(sortedRelativeLifts, 0.998);
167
+ const span = upper - lower;
168
+ if (!Number.isFinite(span) || span <= 0) {
169
+ return [
170
+ { relativeLift: lower, density: 0 },
171
+ { relativeLift: lower, density: 1 },
172
+ { relativeLift: lower, density: 0 }
173
+ ];
174
+ }
175
+ const totalA = alphaA + betaA;
176
+ const meanA = alphaA / totalA;
177
+ const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
178
+ const aLow = Math.max(1e-9, meanA - 10 * sdA);
179
+ const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
180
+ const lifts = Array.from(
181
+ { length: LIFT_DENSITY_POINTS },
182
+ (_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
183
+ );
184
+ const densities = lifts.map((relativeLift) => {
185
+ const ratio = 1 + relativeLift;
186
+ if (ratio <= 0) return 0;
187
+ const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
188
+ if (aHigh <= aLow) return 0;
189
+ const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
190
+ let total = 0;
191
+ for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
192
+ const a = aLow + step * index;
193
+ const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
194
+ total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
195
+ }
196
+ return total * step / 3;
197
+ });
198
+ const peak = Math.max(...densities, Number.MIN_VALUE);
199
+ return lifts.map((relativeLift, index) => ({
200
+ relativeLift,
201
+ density: densities[index] / peak
202
+ }));
203
+ }
204
+
161
205
  // src/core/plan.ts
162
206
  var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
163
207
  function requiredSamplePerVariant({
@@ -215,7 +259,7 @@ var PRIOR_ALPHA = 1;
215
259
  var PRIOR_BETA = 1;
216
260
  var POSTERIOR_CHART_POINTS = 96;
217
261
  var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
218
- var WAITING_VALUE_SIMULATIONS = 1500;
262
+ var WAITING_VALUE_SIMULATIONS = 6e3;
219
263
  var DECISION_LABELS = {
220
264
  ship_b: "Ship Variant B",
221
265
  keep_a: "Keep Variant A",
@@ -276,8 +320,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
276
320
  });
277
321
  }
278
322
  function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
279
- const random = makeDeterministicRandom(alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19);
323
+ const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
280
324
  return WAITING_SCENARIOS.map((extraPerVariant) => {
325
+ const random = makeDeterministicRandom(seed);
281
326
  let regretSum = 0;
282
327
  for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
283
328
  const trueRateA = sampleBeta(alphaA, betaA, random);
@@ -361,7 +406,9 @@ function analyze({
361
406
  aBeatsB: 1 - pBBeatsA,
362
407
  meaningfulLift: probabilityMeaningfulLift,
363
408
  meaningfulHarm: probabilityMeaningfulHarm,
364
- harmful: share((lift) => lift < 0),
409
+ /* The exact complement of the closed-form win probability. Estimating it
410
+ from the samples left the two halves summing to slightly less than 1. */
411
+ harmful: 1 - pBBeatsA,
365
412
  liftAbove1pct: share((lift) => lift > 0.01),
366
413
  liftAbove5pct: share((lift) => lift > 0.05),
367
414
  harmBeyond5pct: share((lift) => lift < -0.05),
@@ -373,7 +420,9 @@ function analyze({
373
420
  absoluteDiff: posteriorMeanB - posteriorMeanA,
374
421
  absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
375
422
  observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
376
- observedAbsoluteDiff: rateB - rateA
423
+ observedAbsoluteDiff: rateB - rateA,
424
+ relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
425
+ relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
377
426
  },
378
427
  expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
379
428
  variants: {
@@ -393,6 +442,7 @@ function analyze({
393
442
  const sortedA = [...samplesA].sort((left, right) => left - right);
394
443
  const sortedB = [...samplesB].sort((left, right) => left - right);
395
444
  result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
445
+ result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
396
446
  }
397
447
  return result;
398
448
  }
@@ -819,7 +869,7 @@ OUTPUT
819
869
  so piping or calling from an agent gets JSON automatically.
820
870
  --pretty Indent the JSON.
821
871
  --brief One-line human summary (analyze and project).
822
- --include-chart Add 96 posterior density points to analyze's JSON.
872
+ --include-chart Add posterior and lift density curves to analyze's JSON.
823
873
  --samples N Monte Carlo samples for intervals and loss. default 12000
824
874
  --seed N RNG seed. Defaults to a value derived from the inputs, so identical
825
875
  inputs always give identical output.
@@ -1217,7 +1267,7 @@ var ANALYZE_SCHEMA = {
1217
1267
  aBeatsB: fraction,
1218
1268
  meaningfulLift: { ...fraction, description: "P(relative lift > minEffect)" },
1219
1269
  meaningfulHarm: { ...fraction, description: "P(relative lift < -minEffect)" },
1220
- harmful: { ...fraction, description: "P(relative lift < 0)" },
1270
+ harmful: { ...fraction, description: "P(B < A), the exact complement of bBeatsA" },
1221
1271
  liftAbove1pct: fraction,
1222
1272
  liftAbove5pct: fraction,
1223
1273
  harmBeyond5pct: fraction,
@@ -1229,7 +1279,9 @@ var ANALYZE_SCHEMA = {
1229
1279
  absoluteDiff: number,
1230
1280
  absoluteDiffCI95: pair(number),
1231
1281
  observedRelativeLift: { type: ["number", "null"] },
1232
- observedAbsoluteDiff: number
1282
+ observedAbsoluteDiff: number,
1283
+ relativeLiftMedian: number,
1284
+ relativeLiftIQR: pair(number)
1233
1285
  }),
1234
1286
  expectedLoss: object({ shipA: fraction, shipB: fraction }),
1235
1287
  variants: object({
@@ -1251,6 +1303,11 @@ var ANALYZE_SCHEMA = {
1251
1303
  description: "Only with --include-chart.",
1252
1304
  items: object({ rate: fraction, densityA: number, densityB: number })
1253
1305
  },
1306
+ liftChart: {
1307
+ type: "array",
1308
+ description: "Only with --include-chart. Exact density of the relative lift, peak scaled to 1.",
1309
+ items: object({ relativeLift: number, density: number })
1310
+ },
1254
1311
  shareUrl: { type: "string", format: "uri" }
1255
1312
  }, [
1256
1313
  "tool",
@@ -1343,7 +1400,7 @@ var SCHEMAS = {
1343
1400
  };
1344
1401
 
1345
1402
  // src/cli/main.ts
1346
- var VERSION = true ? "0.1.0" : "dev";
1403
+ var VERSION = true ? "0.2.0" : "dev";
1347
1404
  var JSON_DECIMALS = 6;
1348
1405
  function roundDeep(value) {
1349
1406
  if (typeof value === "number") return round(value, JSON_DECIMALS);
@@ -1367,7 +1424,7 @@ function envelope2(command, result, derivedFrom2) {
1367
1424
  return { tool: "bayes-ab", version: VERSION, command, ...result, inputs };
1368
1425
  }
1369
1426
  function stripChart(result) {
1370
- const { chart, ...rest } = result;
1427
+ const { chart, liftChart, ...rest } = result;
1371
1428
  return rest;
1372
1429
  }
1373
1430
  function run(argv, io) {
@@ -1,3 +1,4 @@
1
+ import { type LiftDensityPoint } from "./lift.ts";
1
2
  export declare const DEFAULT_THRESHOLD = 0.95;
2
3
  export declare const DEFAULT_MIN_EFFECT = 0.1;
3
4
  export declare const DEFAULT_SAMPLES = 12000;
@@ -75,6 +76,8 @@ export type AnalyzeResult = {
75
76
  absoluteDiffCI95: [number, number];
76
77
  observedRelativeLift: number | null;
77
78
  observedAbsoluteDiff: number;
79
+ relativeLiftMedian: number;
80
+ relativeLiftIQR: [number, number];
78
81
  };
79
82
  expectedLoss: {
80
83
  shipA: number;
@@ -94,6 +97,7 @@ export type AnalyzeResult = {
94
97
  pValueTwoSided: number;
95
98
  };
96
99
  chart?: ChartPoint[];
100
+ liftChart?: LiftDensityPoint[];
97
101
  shareUrl: string;
98
102
  };
99
103
  export declare function deriveSeed(...parts: number[]): number;
@@ -0,0 +1,15 @@
1
+ export type LiftDensityPoint = {
2
+ relativeLift: number;
3
+ /** Scaled so the peak is 1. */
4
+ density: number;
5
+ };
6
+ /**
7
+ * Density of B's relative lift over A, computed exactly rather than
8
+ * histogrammed from the Monte Carlo samples. Writing B = A(1 + r), the ratio
9
+ * density is f(r) = ∫ a · f_A(a) · f_B(a(1+r)) da, which is one Simpson
10
+ * integral per grid point. A histogram of 12k draws carries ~10% sampling
11
+ * noise per bin, and on a wide posterior that reads as a jagged curve.
12
+ *
13
+ * The sorted Monte Carlo lifts only set the plotted range (0.2% to 99.8%).
14
+ */
15
+ export declare function liftDensity(alphaA: number, betaA: number, alphaB: number, betaB: number, sortedRelativeLifts: number[]): LiftDensityPoint[];
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Expected loss of each choice by direct numerical integration over a 600-step
3
+ * grid, with no sampling. Deterministic and smooth in its inputs, so it suits
4
+ * curves that sweep sample size; `analyze` uses Monte Carlo instead because it
5
+ * shares those draws with the lift interval and threshold probabilities.
6
+ *
7
+ * Losses are absolute conversion-rate differences (0.001 = 0.1 pp):
8
+ * shipA = E[max(B − A, 0)], shipB = E[max(A − B, 0)].
9
+ */
10
+ export declare function integratedExpectedLoss(alphaA: number, betaA: number, alphaB: number, betaB: number): {
11
+ shipA: number;
12
+ shipB: number;
13
+ };
package/dist/index.d.ts CHANGED
@@ -1,8 +1,11 @@
1
- export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD } from "./core/analyze.ts";
1
+ export { analyze, DECISION_LABELS, DEFAULT_MIN_EFFECT, DEFAULT_SAMPLES, DEFAULT_THRESHOLD, PRIOR_ALPHA, PRIOR_BETA } from "./core/analyze.ts";
2
2
  export type { AnalyzeInputs, AnalyzeResult, ChartPoint, DecisionStatus, EvidenceStrength, Maturity, Variant, VariantSummary, WaitingScenario, } from "./core/analyze.ts";
3
3
  export { project, DEFAULT_HORIZON_DAYS, DEFAULT_MAX_LOSS, DEFAULT_SIMULATIONS } from "./core/project.ts";
4
4
  export type { ProjectInputs, ProjectResult, ProjectedDecision, StopRule } from "./core/project.ts";
5
5
  export { plan, requiredSamplePerVariant } from "./core/plan.ts";
6
6
  export type { PlanInputs, PlanResult, PlanRow } from "./core/plan.ts";
7
7
  export { shareUrl, WEB_TOOL_URL } from "./core/share.ts";
8
- export { probabilityBBeatsA } from "./core/math.ts";
8
+ export { betaPdf, logBeta, logGamma, probabilityBBeatsA } from "./core/math.ts";
9
+ export { liftDensity } from "./core/lift.ts";
10
+ export type { LiftDensityPoint } from "./core/lift.ts";
11
+ export { integratedExpectedLoss } from "./core/loss.ts";
package/dist/index.js CHANGED
@@ -151,6 +151,50 @@ function normalPdf(x) {
151
151
  return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
152
152
  }
153
153
 
154
+ // src/core/lift.ts
155
+ var LIFT_DENSITY_POINTS = 160;
156
+ var LIFT_INTEGRATION_STEPS = 400;
157
+ function liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts) {
158
+ const lower = quantile(sortedRelativeLifts, 2e-3);
159
+ const upper = quantile(sortedRelativeLifts, 0.998);
160
+ const span = upper - lower;
161
+ if (!Number.isFinite(span) || span <= 0) {
162
+ return [
163
+ { relativeLift: lower, density: 0 },
164
+ { relativeLift: lower, density: 1 },
165
+ { relativeLift: lower, density: 0 }
166
+ ];
167
+ }
168
+ const totalA = alphaA + betaA;
169
+ const meanA = alphaA / totalA;
170
+ const sdA = Math.sqrt(alphaA * betaA / (totalA * totalA * (totalA + 1)));
171
+ const aLow = Math.max(1e-9, meanA - 10 * sdA);
172
+ const aHighBase = Math.min(1 - 1e-9, meanA + 10 * sdA);
173
+ const lifts = Array.from(
174
+ { length: LIFT_DENSITY_POINTS },
175
+ (_, index) => lower + span * index / (LIFT_DENSITY_POINTS - 1)
176
+ );
177
+ const densities = lifts.map((relativeLift) => {
178
+ const ratio = 1 + relativeLift;
179
+ if (ratio <= 0) return 0;
180
+ const aHigh = Math.min(aHighBase, (1 - 1e-9) / ratio);
181
+ if (aHigh <= aLow) return 0;
182
+ const step = (aHigh - aLow) / LIFT_INTEGRATION_STEPS;
183
+ let total = 0;
184
+ for (let index = 0; index <= LIFT_INTEGRATION_STEPS; index += 1) {
185
+ const a = aLow + step * index;
186
+ const weight = index === 0 || index === LIFT_INTEGRATION_STEPS ? 1 : index % 2 ? 4 : 2;
187
+ total += weight * a * betaPdf(a, alphaA, betaA) * betaPdf(a * ratio, alphaB, betaB);
188
+ }
189
+ return total * step / 3;
190
+ });
191
+ const peak = Math.max(...densities, Number.MIN_VALUE);
192
+ return lifts.map((relativeLift, index) => ({
193
+ relativeLift,
194
+ density: densities[index] / peak
195
+ }));
196
+ }
197
+
154
198
  // src/core/plan.ts
155
199
  var SENSITIVITY_EFFECTS = [0.05, 0.1, 0.15, 0.2];
156
200
  function requiredSamplePerVariant({
@@ -208,7 +252,7 @@ var PRIOR_ALPHA = 1;
208
252
  var PRIOR_BETA = 1;
209
253
  var POSTERIOR_CHART_POINTS = 96;
210
254
  var WAITING_SCENARIOS = [1e3, 2e3, 5e3, 1e4];
211
- var WAITING_VALUE_SIMULATIONS = 1500;
255
+ var WAITING_VALUE_SIMULATIONS = 6e3;
212
256
  var DECISION_LABELS = {
213
257
  ship_b: "Ship Variant B",
214
258
  keep_a: "Keep Variant A",
@@ -269,8 +313,9 @@ function chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB) {
269
313
  });
270
314
  }
271
315
  function waitingScenarios(alphaA, betaA, alphaB, betaB, expectedRegretNow) {
272
- const random = makeDeterministicRandom(alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19);
316
+ const seed = alphaA * 11 + betaA * 13 + alphaB * 17 + betaB * 19;
273
317
  return WAITING_SCENARIOS.map((extraPerVariant) => {
318
+ const random = makeDeterministicRandom(seed);
274
319
  let regretSum = 0;
275
320
  for (let index = 0; index < WAITING_VALUE_SIMULATIONS; index += 1) {
276
321
  const trueRateA = sampleBeta(alphaA, betaA, random);
@@ -354,7 +399,9 @@ function analyze({
354
399
  aBeatsB: 1 - pBBeatsA,
355
400
  meaningfulLift: probabilityMeaningfulLift,
356
401
  meaningfulHarm: probabilityMeaningfulHarm,
357
- harmful: share((lift) => lift < 0),
402
+ /* The exact complement of the closed-form win probability. Estimating it
403
+ from the samples left the two halves summing to slightly less than 1. */
404
+ harmful: 1 - pBBeatsA,
358
405
  liftAbove1pct: share((lift) => lift > 0.01),
359
406
  liftAbove5pct: share((lift) => lift > 0.05),
360
407
  harmBeyond5pct: share((lift) => lift < -0.05),
@@ -366,7 +413,9 @@ function analyze({
366
413
  absoluteDiff: posteriorMeanB - posteriorMeanA,
367
414
  absoluteDiffCI95: [quantile(sortedAbsoluteDiffs, 0.025), quantile(sortedAbsoluteDiffs, 0.975)],
368
415
  observedRelativeLift: rateA > 0 ? rateB / rateA - 1 : null,
369
- observedAbsoluteDiff: rateB - rateA
416
+ observedAbsoluteDiff: rateB - rateA,
417
+ relativeLiftMedian: quantile(sortedRelativeLifts, 0.5),
418
+ relativeLiftIQR: [quantile(sortedRelativeLifts, 0.25), quantile(sortedRelativeLifts, 0.75)]
370
419
  },
371
420
  expectedLoss: { shipA: expectedLossShipA, shipB: expectedLossShipB },
372
421
  variants: {
@@ -386,6 +435,7 @@ function analyze({
386
435
  const sortedA = [...samplesA].sort((left, right) => left - right);
387
436
  const sortedB = [...samplesB].sort((left, right) => left - right);
388
437
  result.chart = chartData(alphaA, betaA, alphaB, betaB, sortedA, sortedB);
438
+ result.liftChart = liftDensity(alphaA, betaA, alphaB, betaB, sortedRelativeLifts);
389
439
  }
390
440
  return result;
391
441
  }
@@ -562,6 +612,30 @@ function project({
562
612
  shareUrl: link
563
613
  };
564
614
  }
615
+
616
+ // src/core/loss.ts
617
+ function integratedExpectedLoss(alphaA, betaA, alphaB, betaB) {
618
+ const GRID = 600;
619
+ const dx = 1 / GRID;
620
+ let cdfA = 0;
621
+ let cdfB = 0;
622
+ let weightedSumA = 0;
623
+ let weightedSumB = 0;
624
+ let shipB = 0;
625
+ let shipA = 0;
626
+ for (let i = 0; i <= GRID; i += 1) {
627
+ const x = clamp(i * dx, 1e-6, 1 - 1e-6);
628
+ const fA = betaPdf(x, alphaA, betaA);
629
+ const fB = betaPdf(x, alphaB, betaB);
630
+ shipB += fA * (x * cdfB - weightedSumB) * dx;
631
+ shipA += fB * (x * cdfA - weightedSumA) * dx;
632
+ cdfA += fA * dx;
633
+ weightedSumA += x * fA * dx;
634
+ cdfB += fB * dx;
635
+ weightedSumB += x * fB * dx;
636
+ }
637
+ return { shipA: Math.max(shipA, 0), shipB: Math.max(shipB, 0) };
638
+ }
565
639
  export {
566
640
  DECISION_LABELS,
567
641
  DEFAULT_HORIZON_DAYS,
@@ -570,8 +644,15 @@ export {
570
644
  DEFAULT_SAMPLES,
571
645
  DEFAULT_SIMULATIONS,
572
646
  DEFAULT_THRESHOLD,
647
+ PRIOR_ALPHA,
648
+ PRIOR_BETA,
573
649
  WEB_TOOL_URL,
574
650
  analyze,
651
+ betaPdf,
652
+ integratedExpectedLoss,
653
+ liftDensity,
654
+ logBeta,
655
+ logGamma,
575
656
  plan,
576
657
  probabilityBBeatsA,
577
658
  project,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bayes-ab",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "Bayesian A/B test analysis on the command line, for humans and agents",
5
5
  "keywords": ["ab-testing", "bayesian", "statistics", "cli", "experimentation", "conversion-rate"],
6
6
  "homepage": "https://www.joebrundage.com/tools/bayesian-ab-test",