@splithub/node 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -23,10 +23,31 @@ Routes relative to the mount path:
23
23
  | Method | Path | Purpose |
24
24
  |---|---|---|
25
25
  | POST | `/` | Ingest an event batch from the SDK |
26
- | GET | `/status` | All experiments with counts and a verdict headline |
27
- | GET | `/results/:id` | Full verdict for one experiment |
26
+ | GET | `/status` | Every experiment with exposures, visitors and days |
27
+ | GET | `/results/:id` | Visitors, conversions, rate (and revenue) per variant |
28
28
  | GET | `/health` | Liveness |
29
29
 
30
+ Reads return raw numbers from your own database:
31
+
32
+ ```json
33
+ {
34
+ "experiment": "hero",
35
+ "kind": "component",
36
+ "status": "running",
37
+ "goal": "signup",
38
+ "variants": [
39
+ { "variant": "control", "visitors": 1204, "conversions": 61, "rate": 0.0507 },
40
+ { "variant": "bold", "visitors": 1189, "conversions": 79, "rate": 0.0664 }
41
+ ],
42
+ "metrics": [{ "metric": "refund", "role": "guardrail", "worse": "higher", "variants": [] }],
43
+ "days": 6.2,
44
+ "analysis": { "available": false, "message": "Verdicts, stars and recommendations come with SplitHub Cloud.", "url": "https://splithub.io/cloud" }
45
+ }
46
+ ```
47
+
48
+ The first variant is the control. `rawResults(config, store, id)` and `rawStatusTable(config, store)`
49
+ give the same numbers in code. Verdicts, data-quality stars and recommendations come with SplitHub Cloud.
50
+
30
51
  ## Bring your own database
31
52
 
32
53
  Implement the `Store` interface (three methods) against Postgres, Supabase,
@@ -35,7 +56,7 @@ Prisma, Mongo, anything:
35
56
  ```ts
36
57
  interface Store {
37
58
  insertEvents(events: StoredEvent[]): Promise<number>; // ignore duplicate ids
38
- variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>;
59
+ variantCounts(experiment: string, goal: string): Promise<StoreVariantCounts[]>;
39
60
  summaries(): Promise<ExperimentSummary[]>;
40
61
  }
41
62
  ```
package/dist/index.cjs CHANGED
@@ -20,22 +20,19 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
+ ANALYSIS_UNAVAILABLE: () => ANALYSIS_UNAVAILABLE,
23
24
  ATTRIBUTION_SKEW_MS: () => ATTRIBUTION_SKEW_MS,
24
25
  GOAL_DAILY_CAP: () => GOAL_DAILY_CAP,
25
26
  attributeVariants: () => attributeVariants,
26
27
  attributionConformance: () => attributionConformance,
27
- bestOf: () => bestOf,
28
28
  countsFromRow: () => countsFromRow,
29
29
  createMemoryStore: () => createMemoryStore,
30
30
  createSplitHubHandler: () => createSplitHubHandler,
31
- experimentResults: () => experimentResults,
32
- factorEffects: () => factorEffects,
33
- flowExpectedShares: () => flowExpectedShares,
34
- headlineFor: () => headlineFor,
31
+ rawResults: () => rawResults,
32
+ rawStatusTable: () => rawStatusTable,
35
33
  resultsExperiment: () => resultsExperiment,
36
34
  revenueOf: () => revenueOf,
37
35
  sortCounts: () => sortCounts,
38
- statusTable: () => statusTable,
39
36
  toNodeListener: () => toNodeListener,
40
37
  validateBatch: () => validateBatch,
41
38
  variantCountsSql: () => variantCountsSql
@@ -45,221 +42,68 @@ module.exports = __toCommonJS(index_exports);
45
42
  // src/handler.ts
46
43
  var import_core3 = require("@splithub/core");
47
44
 
48
- // src/results.ts
45
+ // src/raw.ts
49
46
  var import_core = require("@splithub/core");
50
- var import_stats = require("@splithub/stats");
51
47
  var DAY_MS = 864e5;
52
- function factorLevels(experiment, factor, counts2) {
53
- return factor.levels.map((level) => {
54
- const combos = experiment.variants.filter((v) => v.levels?.[factor.key] === level).map((v) => v.key);
55
- const rows = counts2.filter((c) => combos.includes(c.variant));
56
- return { variant: level, exposures: rows.reduce((s, r) => s + r.exposures, 0), conversions: rows.reduce((s, r) => s + r.conversions, 0) };
57
- });
58
- }
59
- function factorPValues(experiment, counts2) {
60
- const out = [];
61
- for (const factor of experiment.factors ?? []) {
62
- const perLevel = factorLevels(experiment, factor, counts2);
63
- if (perLevel.every((l) => l.exposures > 0)) out.push(...(0, import_stats.compareVariants)(perLevel).map((c) => c.sequential.pValue));
64
- }
65
- return out;
66
- }
67
- function factorEffects(experiment, counts2, options = {}) {
68
- if (!experiment.factors?.length) return [];
69
- const perFactor = experiment.factors.map((factor) => {
70
- const perLevel = factorLevels(experiment, factor, counts2);
71
- return { factor, perLevel, comparisons: perLevel.every((l) => l.exposures > 0) ? (0, import_stats.compareVariants)(perLevel) : [] };
72
- });
73
- const all = perFactor.flatMap((f) => f.comparisons);
74
- const rejected = (0, import_stats.holm)([...all.map((c) => c.sequential.pValue), ...options.familyPValues ?? []], 1 - import_stats.VERDICT_DEFAULTS.confidence);
75
- const significant = new Map(all.map((c, i) => [c, rejected[i]]));
76
- return perFactor.map(({ factor, perLevel, comparisons }) => {
77
- const levels = perLevel.map((l, i) => {
78
- const cmp = i === 0 ? void 0 : comparisons.find((c) => c.variant === l.variant);
79
- return {
80
- level: l.variant,
81
- exposures: l.exposures,
82
- conversions: l.conversions,
83
- rate: l.exposures ? l.conversions / l.exposures : 0,
84
- relativeLift: cmp?.relativeLift ?? null,
85
- pValue: cmp?.pValue ?? null,
86
- sequentialPValue: cmp?.sequential.pValue ?? null,
87
- probabilityToBeat: cmp?.probabilityToBeat ?? null,
88
- significant: cmp ? Boolean(significant.get(cmp)) : false
89
- };
90
- });
91
- const winners = levels.filter((l) => l.significant && (l.relativeLift ?? 0) > 0).sort((a, b) => (b.relativeLift ?? 0) - (a.relativeLift ?? 0));
92
- return { key: factor.key, levels, best: winners[0]?.level ?? null };
93
- });
94
- }
95
- var sum = (xs) => xs.reduce((s, x) => s + x, 0);
96
- var sameShares = (a, b) => {
97
- const sa = sum(a);
98
- const sb = sum(b);
99
- return a.every((x, i) => Math.abs(x / sa - b[i] / sb) < 1e-9);
48
+ var ANALYSIS_UNAVAILABLE = {
49
+ available: false,
50
+ message: "Verdicts, stars and recommendations come with SplitHub Cloud.",
51
+ url: "https://splithub.io/cloud"
100
52
  };
101
- function flowExpectedShares(experiment, flow) {
102
- const keys = experiment.variants.map((v) => v.key);
103
- const zero = () => new Array(keys.length).fill(0);
104
- const memo = /* @__PURE__ */ new Map();
105
- const walk = (id, depth) => {
106
- if (!id) return { mass: zero(), known: true };
107
- if (depth > 64) return "unknown";
108
- const cached = memo.get(id);
109
- if (cached) return cached;
110
- const node = flow.nodes[id];
111
- let out;
112
- if (!node || node.type === "skip") out = { mass: zero(), known: true };
113
- else if (node.type === "variant") {
114
- const mass = zero();
115
- const i = keys.indexOf(node.variant);
116
- if (i >= 0) mass[i] = 1;
117
- out = { mass, known: true };
118
- } else {
119
- const raw = node.type === "split" ? node.outputs.filter((o) => o.weight > 0).map((o) => ({ weight: o.weight, reach: walk(o.to, depth + 1) })) : [{ weight: 1, reach: walk(node.then, depth + 1) }, { weight: 1, reach: walk(node.else, depth + 1) }];
120
- const branches = [];
121
- for (const b of raw) if (b.reach !== "unknown") branches.push({ weight: b.weight, reach: b.reach });
122
- const live = branches.filter((b) => sum(b.reach.mass) > 0);
123
- if (branches.length < raw.length) out = "unknown";
124
- else if (node.type === "split" && branches.every((b) => b.reach.known)) {
125
- const total = sum(branches.map((b) => b.weight));
126
- const mass = zero();
127
- for (const b of branches) b.reach.mass.forEach((m, i) => mass[i] += b.weight / total * m);
128
- out = { mass, known: true };
129
- } else if (live.length === 0) out = { mass: zero(), known: true };
130
- else if (live.every((b) => sameShares(b.reach.mass, live[0].reach.mass))) {
131
- const ref = live[0].reach.mass;
132
- out = { mass: ref.map((m) => m / sum(ref)), known: false };
133
- } else out = "unknown";
134
- }
135
- memo.set(id, out);
136
- return out;
137
- };
138
- const reach = walk(flow.start, 0);
139
- if (reach === "unknown" || sum(reach.mass) <= 0) return null;
140
- return reach.mass;
141
- }
142
53
  function resultsExperiment(config, key) {
143
54
  return (0, import_core.getExperiment)(config, key) ?? (key === import_core.HOLDOUT_KEY ? (0, import_core.holdoutExperiment)(config) : null);
144
55
  }
145
- function alignCounts(experiment, observed) {
56
+ function rawVariants(experiment, observed, withRevenue) {
146
57
  return experiment.variants.map((v) => {
147
58
  const row = observed.find((o) => o.variant === v.key);
59
+ const visitors = row?.exposures ?? 0;
60
+ const conversions = row?.conversions ?? 0;
61
+ const revenue = row?.revenueSum ?? row?.revenue;
148
62
  return {
149
63
  variant: v.key,
150
- exposures: row?.exposures ?? 0,
151
- conversions: row?.conversions ?? 0,
152
- revenue: row?.revenue ?? row?.revenueSum ?? 0,
153
- ...row?.revenueSum !== void 0 ? { revenueSum: row.revenueSum } : {},
154
- ...row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
64
+ visitors,
65
+ conversions,
66
+ rate: visitors > 0 ? conversions / visitors : 0,
67
+ ...withRevenue && revenue !== void 0 ? { revenue } : {},
68
+ ...withRevenue && row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
155
69
  };
156
70
  });
157
71
  }
158
- async function metricInputs(experiment, store) {
159
- const m = experiment.metrics;
160
- if (!m) return [];
161
- const list = [
162
- ...m.secondary.map((metric) => ({ metric, role: "secondary" })),
163
- ...m.guardrails.map((g) => ({ metric: g.goal, role: "guardrail", worse: g.worse }))
164
- ];
165
- return Promise.all(list.map(async (x) => ({ ...x, counts: alignCounts(experiment, await store.variantCounts(experiment.key, x.metric)) })));
166
- }
167
- async function experimentResults(config, store, key, options = {}) {
72
+ async function rawResults(config, store, key, options = {}) {
168
73
  const experiment = resultsExperiment(config, key);
169
74
  if (!experiment) return null;
170
75
  const goal2 = experiment.goal;
171
76
  const observed = goal2 ? await store.variantCounts(key, goal2) : [];
172
- const counts2 = alignCounts(experiment, observed);
173
- const metrics = goal2 ? await metricInputs(experiment, store) : [];
174
- const now = options.now ?? /* @__PURE__ */ new Date();
175
- const summary = options.summary !== void 0 ? options.summary : goal2 ? (await store.summaries()).find((s) => s.experiment === key) ?? null : null;
176
- const span = summary?.firstExposureAt != null && summary.lastExposureAt != null ? (summary.lastExposureAt - summary.firstExposureAt) / DAY_MS : void 0;
177
- const days = options.days ?? span;
178
- const visitors = counts2.reduce((s, c) => s + c.exposures, 0);
179
- const visitorsPerDay = options.visitorsPerDay ?? (days !== void 0 && days >= 1 / 24 && visitors > 0 ? visitors / days : null);
180
- let expected = { expectedWeights: experiment.variants.map((v) => v.weight) };
181
- if (experiment.flow) {
182
- const shares = flowExpectedShares(experiment, experiment.flow);
183
- expected = shares ? { expectedWeights: shares } : { skipSrm: "Split check skipped: targeting conditions decide which variant a visitor reaches, so the expected split depends on who visits." };
184
- }
185
- const isMvt = experiment.kind === "mvt" && Boolean(goal2);
186
- const familyPValues = isMvt ? factorPValues(experiment, counts2) : void 0;
187
- const limits = { ...experiment.minDays !== void 0 ? { minDays: experiment.minDays } : {}, ...experiment.goodSample !== void 0 ? { goodSample: experiment.goodSample } : {} };
188
- const verdict = goal2 ? (0, import_stats.computeVerdict)(key, counts2, { ...expected, days, familyPValues, now, ...limits, ...metrics.length ? { metrics } : {} }) : { ...(0, import_stats.computeVerdict)(key, [], { now, ...limits }), message: "No goal configured for this experiment; add `goal` in splithub.config.ts." };
189
- const factors = isMvt ? factorEffects(experiment, counts2, { familyPValues: verdict.comparisons.map((c) => c.sequential.pValue) }) : void 0;
190
- const revenue = goal2 ? revenueResult(counts2, Boolean(config.goals.find((g) => g.key === goal2)?.revenue)) : void 0;
77
+ const goalHasRevenue = Boolean(goal2 && config.goals.find((g) => g.key === goal2)?.revenue);
78
+ const withRevenue = goalHasRevenue || observed.some((o) => (o.revenueSum ?? o.revenue ?? 0) !== 0);
79
+ const summary = options.summary !== void 0 ? options.summary : (await store.summaries()).find((s) => s.experiment === key) ?? null;
80
+ const span = summary?.firstExposureAt != null && summary.lastExposureAt != null ? (summary.lastExposureAt - summary.firstExposureAt) / DAY_MS : 0;
81
+ const m = goal2 ? experiment.metrics : void 0;
82
+ const metrics = m ? await Promise.all([
83
+ ...m.secondary.map(async (metric) => ({ metric, role: "secondary", variants: rawVariants(experiment, await store.variantCounts(key, metric), false) })),
84
+ ...m.guardrails.map(async (g) => ({ metric: g.goal, role: "guardrail", ...g.worse ? { worse: g.worse } : {}, variants: rawVariants(experiment, await store.variantCounts(key, g.goal), false) }))
85
+ ]) : void 0;
191
86
  return {
192
87
  experiment: key,
193
88
  kind: experiment.kind,
194
89
  status: experiment.status,
195
90
  goal: goal2,
196
- description: experiment.description,
197
- verdict,
198
- daysToVerdict: goal2 ? (0, import_stats.estimateDaysToVerdict)(verdict, visitorsPerDay) : null,
199
- visitorsPerDay: visitorsPerDay === null ? null : Math.round(visitorsPerDay * 10) / 10,
200
- ...factors ? { factors } : {},
201
- ...revenue ? { revenue } : {}
202
- };
203
- }
204
- function revenueResult(counts2, goalHasRevenue) {
205
- if (!counts2.every((c) => typeof c.revenueSumSq === "number")) return void 0;
206
- const rows = counts2.map((c) => ({ variant: c.variant, n: c.exposures, sum: c.revenueSum ?? c.revenue ?? 0, sumSq: c.revenueSumSq ?? 0 }));
207
- if (!goalHasRevenue && rows.every((r) => r.sum === 0)) return void 0;
208
- return {
209
- variants: rows.map((r) => ({ variant: r.variant, visitors: r.n, revenue: r.sum, perVisitor: r.n > 0 ? r.sum / r.n : 0 })),
210
- comparisons: (0, import_stats.compareMeans)(rows)
91
+ description: experiment.description ?? null,
92
+ variants: rawVariants(experiment, observed, withRevenue),
93
+ ...metrics?.length ? { metrics } : {},
94
+ days: Math.round(span * 10) / 10,
95
+ analysis: ANALYSIS_UNAVAILABLE
211
96
  };
212
97
  }
213
- async function statusTable(config, store) {
214
- const summaries = await store.summaries();
215
- const byKey = new Map(summaries.map((s) => [s.experiment, s]));
216
- const rows = [];
98
+ async function rawStatusTable(config, store) {
99
+ const byKey = new Map((await store.summaries()).map((s) => [s.experiment, s]));
217
100
  const holdout = (0, import_core.holdoutExperiment)(config);
218
101
  const all = holdout && byKey.has(import_core.HOLDOUT_KEY) ? [...config.experiments, holdout] : config.experiments;
219
- for (const exp of all) {
102
+ return all.map((exp) => {
220
103
  const s = byKey.get(exp.key);
221
- const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) / 864e5)) : 0;
222
- let verdict = null;
223
- let headline = null;
224
- let best = null;
225
- let sample = null;
226
- let stars = null;
227
- if (exp.goal && s) {
228
- const r = await experimentResults(config, store, exp.key, { summary: s });
229
- verdict = r?.verdict.status ?? null;
230
- headline = r ? headlineFor(r.verdict) : null;
231
- best = r ? bestOf(r.verdict) : null;
232
- sample = r?.verdict.sample.quality ?? null;
233
- stars = r?.verdict.sample.stars ?? null;
234
- }
235
- rows.push({ experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days, verdict, headline, best, sample, stars });
236
- }
237
- return rows;
238
- }
239
- function bestOf(v) {
240
- const best = v.comparisons.slice().sort((a, b) => b.absoluteLift - a.absoluteLift)[0];
241
- const name = v.winner ?? best?.variant;
242
- if (!name) return null;
243
- const cmp = best && best.variant === name ? best : v.comparisons.find((c) => c.variant === name);
244
- return { variant: name, index: v.sample.variants.findIndex((x) => x.variant === name) + 1, lift: cmp?.relativeLift ?? null };
245
- }
246
- function headlineFor(v) {
247
- const best = bestOf(v);
248
- const lift = best?.lift != null ? `${best.lift >= 0 ? "+" : ""}${(best.lift * 100).toFixed(1)}%` : "";
249
- switch (v.status) {
250
- case "winner_found":
251
- return `#${best?.index ?? "?"} wins ${lift}`.trim();
252
- case "insufficient_sample":
253
- return best ? `#${best.index} ${lift} (too early)` : "too early";
254
- case "srm_detected":
255
- return "SRM! check setup";
256
- case "guardrail_breached":
257
- return `guardrail hit: ${v.guardrailBreached?.[0]?.metric ?? "?"}`;
258
- case "no_significant_difference":
259
- return "no difference";
260
- default:
261
- return "no data";
262
- }
104
+ const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) / DAY_MS)) : 0;
105
+ return { experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days };
106
+ });
263
107
  }
264
108
 
265
109
  // src/validate.ts
@@ -355,9 +199,9 @@ function createSplitHubHandler(options) {
355
199
  if (request.method === "GET") {
356
200
  if (tail[1] === "health") return json(request, { ok: true, project: config.project });
357
201
  if (!authorized(request)) return json(request, { error: "unauthorized" }, 401);
358
- if (tail[1] === "status") return json(request, { project: config.project, experiments: await statusTable(config, store) });
202
+ if (tail[1] === "status") return json(request, { project: config.project, experiments: await rawStatusTable(config, store), analysis: ANALYSIS_UNAVAILABLE });
359
203
  if (tail[0] === "results" && tail[1]) {
360
- const r = await experimentResults(config, store, tail[1]);
204
+ const r = await rawResults(config, store, tail[1]);
361
205
  return r ? json(request, r) : json(request, { error: `unknown experiment "${tail[1]}"` }, 404);
362
206
  }
363
207
  }
@@ -677,22 +521,19 @@ function sortCounts(rows) {
677
521
  }
678
522
  // Annotate the CommonJS export names for ESM import in node:
679
523
  0 && (module.exports = {
524
+ ANALYSIS_UNAVAILABLE,
680
525
  ATTRIBUTION_SKEW_MS,
681
526
  GOAL_DAILY_CAP,
682
527
  attributeVariants,
683
528
  attributionConformance,
684
- bestOf,
685
529
  countsFromRow,
686
530
  createMemoryStore,
687
531
  createSplitHubHandler,
688
- experimentResults,
689
- factorEffects,
690
- flowExpectedShares,
691
- headlineFor,
532
+ rawResults,
533
+ rawStatusTable,
692
534
  resultsExperiment,
693
535
  revenueOf,
694
536
  sortCounts,
695
- statusTable,
696
537
  toNodeListener,
697
538
  validateBatch,
698
539
  variantCountsSql