@splithub/node 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -23,10 +23,31 @@ Routes relative to the mount path:
23
23
  | Method | Path | Purpose |
24
24
  |---|---|---|
25
25
  | POST | `/` | Ingest an event batch from the SDK |
26
- | GET | `/status` | All experiments with counts and a verdict headline |
27
- | GET | `/results/:id` | Full verdict for one experiment |
26
+ | GET | `/status` | Every experiment with exposures, visitors and days |
27
+ | GET | `/results/:id` | Visitors, conversions, rate (and revenue) per variant |
28
28
  | GET | `/health` | Liveness |
29
29
 
30
+ Reads return raw numbers from your own database:
31
+
32
+ ```json
33
+ {
34
+ "experiment": "hero",
35
+ "kind": "component",
36
+ "status": "running",
37
+ "goal": "signup",
38
+ "variants": [
39
+ { "variant": "control", "visitors": 1204, "conversions": 61, "rate": 0.0507 },
40
+ { "variant": "bold", "visitors": 1189, "conversions": 79, "rate": 0.0664 }
41
+ ],
42
+ "metrics": [{ "metric": "refund", "role": "guardrail", "worse": "higher", "variants": [] }],
43
+ "days": 6.2,
44
+ "analysis": { "available": false, "message": "Verdicts, stars and recommendations come with SplitHub Cloud.", "url": "https://splithub.io/cloud" }
45
+ }
46
+ ```
47
+
48
+ The first variant is the control. `rawResults(config, store, id)` and `rawStatusTable(config, store)`
49
+ give the same numbers in code. Verdicts, data-quality stars and recommendations come with SplitHub Cloud.
50
+
30
51
  ## Bring your own database
31
52
 
32
53
  Implement the `Store` interface (three methods) against Postgres, Supabase,
@@ -35,7 +56,7 @@ Prisma, Mongo, anything:
35
56
  ```ts
36
57
  interface Store {
37
58
  insertEvents(events: StoredEvent[]): Promise<number>; // ignore duplicate ids
38
- variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>;
59
+ variantCounts(experiment: string, goal: string): Promise<StoreVariantCounts[]>;
39
60
  summaries(): Promise<ExperimentSummary[]>;
40
61
  }
41
62
  ```
package/dist/index.cjs CHANGED
@@ -20,22 +20,19 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
+ ANALYSIS_UNAVAILABLE: () => ANALYSIS_UNAVAILABLE,
23
24
  ATTRIBUTION_SKEW_MS: () => ATTRIBUTION_SKEW_MS,
24
25
  GOAL_DAILY_CAP: () => GOAL_DAILY_CAP,
25
26
  attributeVariants: () => attributeVariants,
26
27
  attributionConformance: () => attributionConformance,
27
- bestOf: () => bestOf,
28
28
  countsFromRow: () => countsFromRow,
29
29
  createMemoryStore: () => createMemoryStore,
30
30
  createSplitHubHandler: () => createSplitHubHandler,
31
- experimentResults: () => experimentResults,
32
- factorEffects: () => factorEffects,
33
- flowExpectedShares: () => flowExpectedShares,
34
- headlineFor: () => headlineFor,
31
+ rawResults: () => rawResults,
32
+ rawStatusTable: () => rawStatusTable,
35
33
  resultsExperiment: () => resultsExperiment,
36
34
  revenueOf: () => revenueOf,
37
35
  sortCounts: () => sortCounts,
38
- statusTable: () => statusTable,
39
36
  toNodeListener: () => toNodeListener,
40
37
  validateBatch: () => validateBatch,
41
38
  variantCountsSql: () => variantCountsSql
@@ -45,216 +42,68 @@ module.exports = __toCommonJS(index_exports);
45
42
  // src/handler.ts
46
43
  var import_core3 = require("@splithub/core");
47
44
 
48
- // src/results.ts
45
+ // src/raw.ts
49
46
  var import_core = require("@splithub/core");
50
- var import_stats = require("@splithub/stats");
51
47
  var DAY_MS = 864e5;
52
- function factorLevels(experiment, factor, counts2) {
53
- return factor.levels.map((level) => {
54
- const combos = experiment.variants.filter((v) => v.levels?.[factor.key] === level).map((v) => v.key);
55
- const rows = counts2.filter((c) => combos.includes(c.variant));
56
- return { variant: level, exposures: rows.reduce((s, r) => s + r.exposures, 0), conversions: rows.reduce((s, r) => s + r.conversions, 0) };
57
- });
58
- }
59
- function factorPValues(experiment, counts2) {
60
- const out = [];
61
- for (const factor of experiment.factors ?? []) {
62
- const perLevel = factorLevels(experiment, factor, counts2);
63
- if (perLevel.every((l) => l.exposures > 0)) out.push(...(0, import_stats.compareVariants)(perLevel).map((c) => c.sequential.pValue));
64
- }
65
- return out;
66
- }
67
- function factorEffects(experiment, counts2, options = {}) {
68
- if (!experiment.factors?.length) return [];
69
- const perFactor = experiment.factors.map((factor) => {
70
- const perLevel = factorLevels(experiment, factor, counts2);
71
- return { factor, perLevel, comparisons: perLevel.every((l) => l.exposures > 0) ? (0, import_stats.compareVariants)(perLevel) : [] };
72
- });
73
- const all = perFactor.flatMap((f) => f.comparisons);
74
- const rejected = (0, import_stats.holm)([...all.map((c) => c.sequential.pValue), ...options.familyPValues ?? []], 1 - import_stats.VERDICT_DEFAULTS.confidence);
75
- const significant = new Map(all.map((c, i) => [c, rejected[i]]));
76
- return perFactor.map(({ factor, perLevel, comparisons }) => {
77
- const levels = perLevel.map((l, i) => {
78
- const cmp = i === 0 ? void 0 : comparisons.find((c) => c.variant === l.variant);
79
- return {
80
- level: l.variant,
81
- exposures: l.exposures,
82
- conversions: l.conversions,
83
- rate: l.exposures ? l.conversions / l.exposures : 0,
84
- relativeLift: cmp?.relativeLift ?? null,
85
- pValue: cmp?.pValue ?? null,
86
- sequentialPValue: cmp?.sequential.pValue ?? null,
87
- probabilityToBeat: cmp?.probabilityToBeat ?? null,
88
- significant: cmp ? Boolean(significant.get(cmp)) : false
89
- };
90
- });
91
- const winners = levels.filter((l) => l.significant && (l.relativeLift ?? 0) > 0).sort((a, b) => (b.relativeLift ?? 0) - (a.relativeLift ?? 0));
92
- return { key: factor.key, levels, best: winners[0]?.level ?? null };
93
- });
94
- }
95
- var sum = (xs) => xs.reduce((s, x) => s + x, 0);
96
- var sameShares = (a, b) => {
97
- const sa = sum(a);
98
- const sb = sum(b);
99
- return a.every((x, i) => Math.abs(x / sa - b[i] / sb) < 1e-9);
48
+ var ANALYSIS_UNAVAILABLE = {
49
+ available: false,
50
+ message: "Verdicts, stars and recommendations come with SplitHub Cloud.",
51
+ url: "https://splithub.io/cloud"
100
52
  };
101
- function flowExpectedShares(experiment, flow) {
102
- const keys = experiment.variants.map((v) => v.key);
103
- const zero = () => new Array(keys.length).fill(0);
104
- const memo = /* @__PURE__ */ new Map();
105
- const walk = (id, depth) => {
106
- if (!id) return { mass: zero(), known: true };
107
- if (depth > 64) return "unknown";
108
- const cached = memo.get(id);
109
- if (cached) return cached;
110
- const node = flow.nodes[id];
111
- let out;
112
- if (!node || node.type === "skip") out = { mass: zero(), known: true };
113
- else if (node.type === "variant") {
114
- const mass = zero();
115
- const i = keys.indexOf(node.variant);
116
- if (i >= 0) mass[i] = 1;
117
- out = { mass, known: true };
118
- } else {
119
- const raw = node.type === "split" ? node.outputs.filter((o) => o.weight > 0).map((o) => ({ weight: o.weight, reach: walk(o.to, depth + 1) })) : [{ weight: 1, reach: walk(node.then, depth + 1) }, { weight: 1, reach: walk(node.else, depth + 1) }];
120
- const branches = [];
121
- for (const b of raw) if (b.reach !== "unknown") branches.push({ weight: b.weight, reach: b.reach });
122
- const live = branches.filter((b) => sum(b.reach.mass) > 0);
123
- if (branches.length < raw.length) out = "unknown";
124
- else if (node.type === "split" && branches.every((b) => b.reach.known)) {
125
- const total = sum(branches.map((b) => b.weight));
126
- const mass = zero();
127
- for (const b of branches) b.reach.mass.forEach((m, i) => mass[i] += b.weight / total * m);
128
- out = { mass, known: true };
129
- } else if (live.length === 0) out = { mass: zero(), known: true };
130
- else if (live.every((b) => sameShares(b.reach.mass, live[0].reach.mass))) {
131
- const ref = live[0].reach.mass;
132
- out = { mass: ref.map((m) => m / sum(ref)), known: false };
133
- } else out = "unknown";
134
- }
135
- memo.set(id, out);
136
- return out;
137
- };
138
- const reach = walk(flow.start, 0);
139
- if (reach === "unknown" || sum(reach.mass) <= 0) return null;
140
- return reach.mass;
141
- }
142
53
  function resultsExperiment(config, key) {
143
54
  return (0, import_core.getExperiment)(config, key) ?? (key === import_core.HOLDOUT_KEY ? (0, import_core.holdoutExperiment)(config) : null);
144
55
  }
145
- function alignCounts(experiment, observed) {
56
+ function rawVariants(experiment, observed, withRevenue) {
146
57
  return experiment.variants.map((v) => {
147
58
  const row = observed.find((o) => o.variant === v.key);
59
+ const visitors = row?.exposures ?? 0;
60
+ const conversions = row?.conversions ?? 0;
61
+ const revenue = row?.revenueSum ?? row?.revenue;
148
62
  return {
149
63
  variant: v.key,
150
- exposures: row?.exposures ?? 0,
151
- conversions: row?.conversions ?? 0,
152
- revenue: row?.revenue ?? row?.revenueSum ?? 0,
153
- ...row?.revenueSum !== void 0 ? { revenueSum: row.revenueSum } : {},
154
- ...row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
64
+ visitors,
65
+ conversions,
66
+ rate: visitors > 0 ? conversions / visitors : 0,
67
+ ...withRevenue && revenue !== void 0 ? { revenue } : {},
68
+ ...withRevenue && row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
155
69
  };
156
70
  });
157
71
  }
158
- async function metricInputs(experiment, store) {
159
- const m = experiment.metrics;
160
- if (!m) return [];
161
- const list = [
162
- ...m.secondary.map((metric) => ({ metric, role: "secondary" })),
163
- ...m.guardrails.map((g) => ({ metric: g.goal, role: "guardrail", worse: g.worse }))
164
- ];
165
- return Promise.all(list.map(async (x) => ({ ...x, counts: alignCounts(experiment, await store.variantCounts(experiment.key, x.metric)) })));
166
- }
167
- async function experimentResults(config, store, key, options = {}) {
72
+ async function rawResults(config, store, key, options = {}) {
168
73
  const experiment = resultsExperiment(config, key);
169
74
  if (!experiment) return null;
170
75
  const goal2 = experiment.goal;
171
76
  const observed = goal2 ? await store.variantCounts(key, goal2) : [];
172
- const counts2 = alignCounts(experiment, observed);
173
- const metrics = goal2 ? await metricInputs(experiment, store) : [];
174
- const now = options.now ?? /* @__PURE__ */ new Date();
175
- const summary = options.summary !== void 0 ? options.summary : goal2 ? (await store.summaries()).find((s) => s.experiment === key) ?? null : null;
176
- const span = summary?.firstExposureAt != null && summary.lastExposureAt != null ? (summary.lastExposureAt - summary.firstExposureAt) / DAY_MS : void 0;
177
- const days = options.days ?? span;
178
- const visitors = counts2.reduce((s, c) => s + c.exposures, 0);
179
- const visitorsPerDay = options.visitorsPerDay ?? (days !== void 0 && days >= 1 / 24 && visitors > 0 ? visitors / days : null);
180
- let expected = { expectedWeights: experiment.variants.map((v) => v.weight) };
181
- if (experiment.flow) {
182
- const shares = flowExpectedShares(experiment, experiment.flow);
183
- expected = shares ? { expectedWeights: shares } : { skipSrm: "Split check skipped: targeting conditions decide which variant a visitor reaches, so the expected split depends on who visits." };
184
- }
185
- const isMvt = experiment.kind === "mvt" && Boolean(goal2);
186
- const familyPValues = isMvt ? factorPValues(experiment, counts2) : void 0;
187
- const verdict = goal2 ? (0, import_stats.computeVerdict)(key, counts2, { ...expected, days, familyPValues, now, ...experiment.minDays !== void 0 ? { minDays: experiment.minDays } : {}, ...metrics.length ? { metrics } : {} }) : { ...(0, import_stats.computeVerdict)(key, [], { now }), message: "No goal configured for this experiment; add `goal` in splithub.config.ts." };
188
- const factors = isMvt ? factorEffects(experiment, counts2, { familyPValues: verdict.comparisons.map((c) => c.sequential.pValue) }) : void 0;
189
- const revenue = goal2 ? revenueResult(counts2, Boolean(config.goals.find((g) => g.key === goal2)?.revenue)) : void 0;
77
+ const goalHasRevenue = Boolean(goal2 && config.goals.find((g) => g.key === goal2)?.revenue);
78
+ const withRevenue = goalHasRevenue || observed.some((o) => (o.revenueSum ?? o.revenue ?? 0) !== 0);
79
+ const summary = options.summary !== void 0 ? options.summary : (await store.summaries()).find((s) => s.experiment === key) ?? null;
80
+ const span = summary?.firstExposureAt != null && summary.lastExposureAt != null ? (summary.lastExposureAt - summary.firstExposureAt) / DAY_MS : 0;
81
+ const m = goal2 ? experiment.metrics : void 0;
82
+ const metrics = m ? await Promise.all([
83
+ ...m.secondary.map(async (metric) => ({ metric, role: "secondary", variants: rawVariants(experiment, await store.variantCounts(key, metric), false) })),
84
+ ...m.guardrails.map(async (g) => ({ metric: g.goal, role: "guardrail", ...g.worse ? { worse: g.worse } : {}, variants: rawVariants(experiment, await store.variantCounts(key, g.goal), false) }))
85
+ ]) : void 0;
190
86
  return {
191
87
  experiment: key,
192
88
  kind: experiment.kind,
193
89
  status: experiment.status,
194
90
  goal: goal2,
195
- description: experiment.description,
196
- verdict,
197
- daysToVerdict: goal2 ? (0, import_stats.estimateDaysToVerdict)(verdict, visitorsPerDay) : null,
198
- visitorsPerDay: visitorsPerDay === null ? null : Math.round(visitorsPerDay * 10) / 10,
199
- ...factors ? { factors } : {},
200
- ...revenue ? { revenue } : {}
201
- };
202
- }
203
- function revenueResult(counts2, goalHasRevenue) {
204
- if (!counts2.every((c) => typeof c.revenueSumSq === "number")) return void 0;
205
- const rows = counts2.map((c) => ({ variant: c.variant, n: c.exposures, sum: c.revenueSum ?? c.revenue ?? 0, sumSq: c.revenueSumSq ?? 0 }));
206
- if (!goalHasRevenue && rows.every((r) => r.sum === 0)) return void 0;
207
- return {
208
- variants: rows.map((r) => ({ variant: r.variant, visitors: r.n, revenue: r.sum, perVisitor: r.n > 0 ? r.sum / r.n : 0 })),
209
- comparisons: (0, import_stats.compareMeans)(rows)
91
+ description: experiment.description ?? null,
92
+ variants: rawVariants(experiment, observed, withRevenue),
93
+ ...metrics?.length ? { metrics } : {},
94
+ days: Math.round(span * 10) / 10,
95
+ analysis: ANALYSIS_UNAVAILABLE
210
96
  };
211
97
  }
212
- async function statusTable(config, store) {
213
- const summaries = await store.summaries();
214
- const byKey = new Map(summaries.map((s) => [s.experiment, s]));
215
- const rows = [];
98
+ async function rawStatusTable(config, store) {
99
+ const byKey = new Map((await store.summaries()).map((s) => [s.experiment, s]));
216
100
  const holdout = (0, import_core.holdoutExperiment)(config);
217
101
  const all = holdout && byKey.has(import_core.HOLDOUT_KEY) ? [...config.experiments, holdout] : config.experiments;
218
- for (const exp of all) {
102
+ return all.map((exp) => {
219
103
  const s = byKey.get(exp.key);
220
- const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) / 864e5)) : 0;
221
- let verdict = null;
222
- let headline = null;
223
- let best = null;
224
- if (exp.goal && s) {
225
- const r = await experimentResults(config, store, exp.key, { summary: s });
226
- verdict = r?.verdict.status ?? null;
227
- headline = r ? headlineFor(r.verdict) : null;
228
- best = r ? bestOf(r.verdict) : null;
229
- }
230
- rows.push({ experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days, verdict, headline, best });
231
- }
232
- return rows;
233
- }
234
- function bestOf(v) {
235
- const best = v.comparisons.slice().sort((a, b) => b.absoluteLift - a.absoluteLift)[0];
236
- const name = v.winner ?? best?.variant;
237
- if (!name) return null;
238
- const cmp = best && best.variant === name ? best : v.comparisons.find((c) => c.variant === name);
239
- return { variant: name, index: v.sample.variants.findIndex((x) => x.variant === name) + 1, lift: cmp?.relativeLift ?? null };
240
- }
241
- function headlineFor(v) {
242
- const best = bestOf(v);
243
- const lift = best?.lift != null ? `${best.lift >= 0 ? "+" : ""}${(best.lift * 100).toFixed(1)}%` : "";
244
- switch (v.status) {
245
- case "winner_found":
246
- return `#${best?.index ?? "?"} wins ${lift}`.trim();
247
- case "insufficient_sample":
248
- return best ? `#${best.index} ${lift} (too early)` : "too early";
249
- case "srm_detected":
250
- return "SRM! check setup";
251
- case "guardrail_breached":
252
- return `guardrail hit: ${v.guardrailBreached?.[0]?.metric ?? "?"}`;
253
- case "no_significant_difference":
254
- return "no difference";
255
- default:
256
- return "no data";
257
- }
104
+ const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) / DAY_MS)) : 0;
105
+ return { experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days };
106
+ });
258
107
  }
259
108
 
260
109
  // src/validate.ts
@@ -350,9 +199,9 @@ function createSplitHubHandler(options) {
350
199
  if (request.method === "GET") {
351
200
  if (tail[1] === "health") return json(request, { ok: true, project: config.project });
352
201
  if (!authorized(request)) return json(request, { error: "unauthorized" }, 401);
353
- if (tail[1] === "status") return json(request, { project: config.project, experiments: await statusTable(config, store) });
202
+ if (tail[1] === "status") return json(request, { project: config.project, experiments: await rawStatusTable(config, store), analysis: ANALYSIS_UNAVAILABLE });
354
203
  if (tail[0] === "results" && tail[1]) {
355
- const r = await experimentResults(config, store, tail[1]);
204
+ const r = await rawResults(config, store, tail[1]);
356
205
  return r ? json(request, r) : json(request, { error: `unknown experiment "${tail[1]}"` }, 404);
357
206
  }
358
207
  }
@@ -672,22 +521,19 @@ function sortCounts(rows) {
672
521
  }
673
522
  // Annotate the CommonJS export names for ESM import in node:
674
523
  0 && (module.exports = {
524
+ ANALYSIS_UNAVAILABLE,
675
525
  ATTRIBUTION_SKEW_MS,
676
526
  GOAL_DAILY_CAP,
677
527
  attributeVariants,
678
528
  attributionConformance,
679
- bestOf,
680
529
  countsFromRow,
681
530
  createMemoryStore,
682
531
  createSplitHubHandler,
683
- experimentResults,
684
- factorEffects,
685
- flowExpectedShares,
686
- headlineFor,
532
+ rawResults,
533
+ rawStatusTable,
687
534
  resultsExperiment,
688
535
  revenueOf,
689
536
  sortCounts,
690
- statusTable,
691
537
  toNodeListener,
692
538
  validateBatch,
693
539
  variantCountsSql