@splithub/node 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/index.cjs +43 -197
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +51 -88
- package/dist/index.d.ts +51 -88
- package/dist/index.js +40 -191
- package/dist/index.js.map +1 -1
- package/package.json +2 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
|
-
import { SplitHubConfig, Experiment
|
|
2
|
-
import { VariantCounts, Verdict, MeanComparison } from '@splithub/stats';
|
|
1
|
+
import { SplitHubConfig, Experiment } from '@splithub/core';
|
|
3
2
|
import { IncomingMessage, ServerResponse } from 'node:http';
|
|
4
3
|
|
|
5
4
|
/** How far a goal may precede the exposure it converts, for clock differences between server and browser. */
|
|
6
5
|
declare const ATTRIBUTION_SKEW_MS = 60000;
|
|
7
6
|
/** Events of one goal from one visitor counted per UTC day. */
|
|
8
7
|
declare const GOAL_DAILY_CAP = 50;
|
|
8
|
+
/** Unique exposed and unique converted visitors of one variant. */
|
|
9
|
+
interface VariantCounts {
|
|
10
|
+
variant: string;
|
|
11
|
+
exposures: number;
|
|
12
|
+
conversions: number;
|
|
13
|
+
revenue?: number;
|
|
14
|
+
revenueSum?: number;
|
|
15
|
+
revenueSumSq?: number;
|
|
16
|
+
}
|
|
9
17
|
/** Per-variant counts every store returns: `revenueSum` / `revenueSumSq` are over per-visitor revenue (non-converted visitors add 0). */
|
|
10
18
|
interface AttributedCounts extends VariantCounts {
|
|
11
19
|
revenue: number;
|
|
@@ -133,8 +141,10 @@ type FetchHandler = (request: Request) => Promise<Response>;
|
|
|
133
141
|
* One handler for ingest and results, mountable at any path:
|
|
134
142
|
*
|
|
135
143
|
* POST {base} → ingest an event batch
|
|
136
|
-
* GET {base}/status → all experiments with
|
|
137
|
-
* GET {base}/results/:id →
|
|
144
|
+
* GET {base}/status → all experiments with exposures, visitors and days
|
|
145
|
+
* GET {base}/results/:id → visitors, conversions, rate (and revenue) per variant
|
|
146
|
+
*
|
|
147
|
+
* Reads are raw counts; verdicts and recommendations are SplitHub Cloud's.
|
|
138
148
|
* GET {base}/health → { ok: true }
|
|
139
149
|
*
|
|
140
150
|
* Works in Next.js route handlers, Remix, Hono, Bun, Deno and Node (via `toNodeListener`).
|
|
@@ -150,85 +160,45 @@ declare function createSplitHubHandler(options: HandlerOptions): FetchHandler;
|
|
|
150
160
|
*/
|
|
151
161
|
declare function toNodeListener(handler: FetchHandler): (req: IncomingMessage, res: ServerResponse) => void;
|
|
152
162
|
|
|
153
|
-
/**
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
163
|
+
/** Self-hosted reads give counts; verdicts and recommendations are computed by SplitHub Cloud. */
|
|
164
|
+
declare const ANALYSIS_UNAVAILABLE: {
|
|
165
|
+
available: false;
|
|
166
|
+
message: string;
|
|
167
|
+
url: string;
|
|
168
|
+
};
|
|
169
|
+
interface RawVariant {
|
|
170
|
+
variant: string;
|
|
171
|
+
/** Unique visitors counted in this variant (first exposure decides). */
|
|
172
|
+
visitors: number;
|
|
173
|
+
/** Visitors who reached the goal after their exposure. */
|
|
157
174
|
conversions: number;
|
|
175
|
+
/** conversions / visitors, 0 without visitors. */
|
|
158
176
|
rate: number;
|
|
159
|
-
/**
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
|
|
163
|
-
/** Always-valid p-value (what `significant` is decided on); null for the first level. */
|
|
164
|
-
sequentialPValue: number | null;
|
|
165
|
-
/** Probability this level beats the first one, same formula as the per-combination table; null for the first level. */
|
|
166
|
-
probabilityToBeat: number | null;
|
|
167
|
-
/** Always-valid, Holm-corrected across every factor level and combination of the experiment. */
|
|
168
|
-
significant: boolean;
|
|
169
|
-
}
|
|
170
|
-
/** A multivariate factor read on its own (its main effect), independent of the other factors. */
|
|
171
|
-
interface FactorResult {
|
|
172
|
-
key: string;
|
|
173
|
-
levels: FactorLevelResult[];
|
|
174
|
-
/** The level that beats the first one significantly with the largest lift, if any. */
|
|
175
|
-
best: string | null;
|
|
177
|
+
/** Sum of per-visitor revenue, when the goal carries revenue. */
|
|
178
|
+
revenue?: number;
|
|
179
|
+
/** Sum of squared per-visitor revenue, when the store reports it. */
|
|
180
|
+
revenueSumSq?: number;
|
|
176
181
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
revenue: number;
|
|
183
|
-
perVisitor: number;
|
|
184
|
-
}>;
|
|
185
|
-
/** Fixed-horizon and always-valid numbers per variant; `significant` uses the always-valid p, Holm-corrected. */
|
|
186
|
-
comparisons: MeanComparison[];
|
|
182
|
+
interface RawMetric {
|
|
183
|
+
metric: string;
|
|
184
|
+
role: 'secondary' | 'guardrail';
|
|
185
|
+
worse?: 'higher' | 'lower';
|
|
186
|
+
variants: RawVariant[];
|
|
187
187
|
}
|
|
188
|
-
interface
|
|
188
|
+
interface RawResults {
|
|
189
189
|
experiment: string;
|
|
190
190
|
kind: string;
|
|
191
191
|
status: string;
|
|
192
192
|
goal: string | null;
|
|
193
193
|
description: string | null;
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
factors?: FactorResult[];
|
|
201
|
-
/** Present when the store reports `revenueSum` and `revenueSumSq` per variant. */
|
|
202
|
-
revenue?: RevenueResult;
|
|
203
|
-
}
|
|
204
|
-
interface ResultsOptions {
|
|
205
|
-
now?: Date;
|
|
206
|
-
/** Days of data; defaults to the span between the first and last exposure in `store.summaries()`. */
|
|
207
|
-
days?: number;
|
|
208
|
-
/** Visitors per day for `daysToVerdict`; defaults to visitors / days. */
|
|
209
|
-
visitorsPerDay?: number;
|
|
210
|
-
/** The experiment's summary when the caller already has it (saves a `summaries()` call). */
|
|
211
|
-
summary?: ExperimentSummary | null;
|
|
194
|
+
/** Declared order: the first variant is the control. */
|
|
195
|
+
variants: RawVariant[];
|
|
196
|
+
metrics?: RawMetric[];
|
|
197
|
+
/** Days between the first and the last exposure, one decimal; 0 without data. */
|
|
198
|
+
days: number;
|
|
199
|
+
analysis: typeof ANALYSIS_UNAVAILABLE;
|
|
212
200
|
}
|
|
213
|
-
|
|
214
|
-
* Main effects of a multivariate experiment: for each factor, every level's
|
|
215
|
-
* visitors and conversions summed over the combinations that use it, compared
|
|
216
|
-
* with the factor's first level. Same tests as the verdict. Significance is
|
|
217
|
-
* Holm-corrected across all factor levels and, when `familyPValues` carries the
|
|
218
|
-
* combinations' always-valid p-values, across those too, so reading factors and
|
|
219
|
-
* combinations together still keeps the 5% false positive rate.
|
|
220
|
-
*/
|
|
221
|
-
declare function factorEffects(experiment: Experiment, counts: VariantCounts[], options?: {
|
|
222
|
-
familyPValues?: number[];
|
|
223
|
-
}): FactorResult[];
|
|
224
|
-
/**
|
|
225
|
-
* Expected traffic shares of the variants under a targeting flow. Splits mix
|
|
226
|
-
* their outputs by weight. A condition sends an unknown share of visitors each
|
|
227
|
-
* way, so the split is only defined when every branch that reaches a variant
|
|
228
|
-
* leads to the same proportions. Null when it depends on who visits.
|
|
229
|
-
*/
|
|
230
|
-
declare function flowExpectedShares(experiment: Experiment, flow: Flow): number[] | null;
|
|
231
|
-
interface StatusRow {
|
|
201
|
+
interface RawStatusRow {
|
|
232
202
|
experiment: string;
|
|
233
203
|
kind: string;
|
|
234
204
|
status: string;
|
|
@@ -236,24 +206,17 @@ interface StatusRow {
|
|
|
236
206
|
description: string | null;
|
|
237
207
|
exposures: number;
|
|
238
208
|
visitors: number;
|
|
209
|
+
/** Whole days with data, at least 1 once there is any; 0 without data. */
|
|
239
210
|
days: number;
|
|
240
|
-
verdict: Verdict['status'] | null;
|
|
241
|
-
/** Short text for tables: "#2 wins +15.9%", "#2 +3.1% (too early)", "no difference", "SRM! check setup", "no data". Ordinals, not names: variant names can be long. */
|
|
242
|
-
headline: string | null;
|
|
243
|
-
/** Leading variant: its name, its ordinal in the declared order (#1 is the control) and the relative lift. */
|
|
244
|
-
best: {
|
|
245
|
-
variant: string;
|
|
246
|
-
index: number;
|
|
247
|
-
lift: number | null;
|
|
248
|
-
} | null;
|
|
249
211
|
}
|
|
250
212
|
/** An experiment of the config, or the holdout comparison for `__holdout`. */
|
|
251
213
|
declare function resultsExperiment(config: SplitHubConfig, key: string): Experiment | null;
|
|
252
|
-
|
|
253
|
-
declare function
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
214
|
+
/** Counts per variant for one experiment, its secondary and guardrail goals included. No statistics. */
|
|
215
|
+
declare function rawResults(config: SplitHubConfig, store: Store, key: string, options?: {
|
|
216
|
+
summary?: ExperimentSummary | null;
|
|
217
|
+
}): Promise<RawResults | null>;
|
|
218
|
+
/** Every experiment with exposures, visitors and days; the holdout once it has visitors. */
|
|
219
|
+
declare function rawStatusTable(config: SplitHubConfig, store: Store): Promise<RawStatusRow[]>;
|
|
257
220
|
|
|
258
221
|
type ValidationResult = {
|
|
259
222
|
ok: true;
|
|
@@ -282,4 +245,4 @@ declare const attributionConformance: ConformanceCase[];
|
|
|
282
245
|
/** Sorts by variant and keeps the compared fields, so any store's output can be `toEqual`-ed with `expected`. */
|
|
283
246
|
declare function sortCounts(rows: StoreVariantCounts[]): AttributedCounts[];
|
|
284
247
|
|
|
285
|
-
export { ATTRIBUTION_SKEW_MS, type AttributedCounts, type AttributionEvent, type ConformanceCase, type
|
|
248
|
+
export { ANALYSIS_UNAVAILABLE, ATTRIBUTION_SKEW_MS, type AttributedCounts, type AttributionEvent, type ConformanceCase, type ExperimentSummary, type FetchHandler, GOAL_DAILY_CAP, type HandlerOptions, type RawMetric, type RawResults, type RawStatusRow, type RawVariant, type SqlDialect, type Store, type StoreVariantCounts, type StoredEvent, type VariantCounts, type VariantCountsSql, type VariantCountsSqlOptions, attributeVariants, attributionConformance, countsFromRow, createMemoryStore, createSplitHubHandler, rawResults, rawStatusTable, resultsExperiment, revenueOf, sortCounts, toNodeListener, validateBatch, variantCountsSql };
|
package/dist/index.js
CHANGED
|
@@ -1,216 +1,68 @@
|
|
|
1
1
|
// src/handler.ts
|
|
2
2
|
import { isCrawler } from "@splithub/core";
|
|
3
3
|
|
|
4
|
-
// src/
|
|
4
|
+
// src/raw.ts
|
|
5
5
|
import { getExperiment, HOLDOUT_KEY, holdoutExperiment } from "@splithub/core";
|
|
6
|
-
import { compareMeans, compareVariants, computeVerdict, estimateDaysToVerdict, holm, VERDICT_DEFAULTS } from "@splithub/stats";
|
|
7
6
|
var DAY_MS = 864e5;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return { variant: level, exposures: rows.reduce((s, r) => s + r.exposures, 0), conversions: rows.reduce((s, r) => s + r.conversions, 0) };
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
function factorPValues(experiment, counts2) {
|
|
16
|
-
const out = [];
|
|
17
|
-
for (const factor of experiment.factors ?? []) {
|
|
18
|
-
const perLevel = factorLevels(experiment, factor, counts2);
|
|
19
|
-
if (perLevel.every((l) => l.exposures > 0)) out.push(...compareVariants(perLevel).map((c) => c.sequential.pValue));
|
|
20
|
-
}
|
|
21
|
-
return out;
|
|
22
|
-
}
|
|
23
|
-
function factorEffects(experiment, counts2, options = {}) {
|
|
24
|
-
if (!experiment.factors?.length) return [];
|
|
25
|
-
const perFactor = experiment.factors.map((factor) => {
|
|
26
|
-
const perLevel = factorLevels(experiment, factor, counts2);
|
|
27
|
-
return { factor, perLevel, comparisons: perLevel.every((l) => l.exposures > 0) ? compareVariants(perLevel) : [] };
|
|
28
|
-
});
|
|
29
|
-
const all = perFactor.flatMap((f) => f.comparisons);
|
|
30
|
-
const rejected = holm([...all.map((c) => c.sequential.pValue), ...options.familyPValues ?? []], 1 - VERDICT_DEFAULTS.confidence);
|
|
31
|
-
const significant = new Map(all.map((c, i) => [c, rejected[i]]));
|
|
32
|
-
return perFactor.map(({ factor, perLevel, comparisons }) => {
|
|
33
|
-
const levels = perLevel.map((l, i) => {
|
|
34
|
-
const cmp = i === 0 ? void 0 : comparisons.find((c) => c.variant === l.variant);
|
|
35
|
-
return {
|
|
36
|
-
level: l.variant,
|
|
37
|
-
exposures: l.exposures,
|
|
38
|
-
conversions: l.conversions,
|
|
39
|
-
rate: l.exposures ? l.conversions / l.exposures : 0,
|
|
40
|
-
relativeLift: cmp?.relativeLift ?? null,
|
|
41
|
-
pValue: cmp?.pValue ?? null,
|
|
42
|
-
sequentialPValue: cmp?.sequential.pValue ?? null,
|
|
43
|
-
probabilityToBeat: cmp?.probabilityToBeat ?? null,
|
|
44
|
-
significant: cmp ? Boolean(significant.get(cmp)) : false
|
|
45
|
-
};
|
|
46
|
-
});
|
|
47
|
-
const winners = levels.filter((l) => l.significant && (l.relativeLift ?? 0) > 0).sort((a, b) => (b.relativeLift ?? 0) - (a.relativeLift ?? 0));
|
|
48
|
-
return { key: factor.key, levels, best: winners[0]?.level ?? null };
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
var sum = (xs) => xs.reduce((s, x) => s + x, 0);
|
|
52
|
-
var sameShares = (a, b) => {
|
|
53
|
-
const sa = sum(a);
|
|
54
|
-
const sb = sum(b);
|
|
55
|
-
return a.every((x, i) => Math.abs(x / sa - b[i] / sb) < 1e-9);
|
|
7
|
+
var ANALYSIS_UNAVAILABLE = {
|
|
8
|
+
available: false,
|
|
9
|
+
message: "Verdicts, stars and recommendations come with SplitHub Cloud.",
|
|
10
|
+
url: "https://splithub.io/cloud"
|
|
56
11
|
};
|
|
57
|
-
function flowExpectedShares(experiment, flow) {
|
|
58
|
-
const keys = experiment.variants.map((v) => v.key);
|
|
59
|
-
const zero = () => new Array(keys.length).fill(0);
|
|
60
|
-
const memo = /* @__PURE__ */ new Map();
|
|
61
|
-
const walk = (id, depth) => {
|
|
62
|
-
if (!id) return { mass: zero(), known: true };
|
|
63
|
-
if (depth > 64) return "unknown";
|
|
64
|
-
const cached = memo.get(id);
|
|
65
|
-
if (cached) return cached;
|
|
66
|
-
const node = flow.nodes[id];
|
|
67
|
-
let out;
|
|
68
|
-
if (!node || node.type === "skip") out = { mass: zero(), known: true };
|
|
69
|
-
else if (node.type === "variant") {
|
|
70
|
-
const mass = zero();
|
|
71
|
-
const i = keys.indexOf(node.variant);
|
|
72
|
-
if (i >= 0) mass[i] = 1;
|
|
73
|
-
out = { mass, known: true };
|
|
74
|
-
} else {
|
|
75
|
-
const raw = node.type === "split" ? node.outputs.filter((o) => o.weight > 0).map((o) => ({ weight: o.weight, reach: walk(o.to, depth + 1) })) : [{ weight: 1, reach: walk(node.then, depth + 1) }, { weight: 1, reach: walk(node.else, depth + 1) }];
|
|
76
|
-
const branches = [];
|
|
77
|
-
for (const b of raw) if (b.reach !== "unknown") branches.push({ weight: b.weight, reach: b.reach });
|
|
78
|
-
const live = branches.filter((b) => sum(b.reach.mass) > 0);
|
|
79
|
-
if (branches.length < raw.length) out = "unknown";
|
|
80
|
-
else if (node.type === "split" && branches.every((b) => b.reach.known)) {
|
|
81
|
-
const total = sum(branches.map((b) => b.weight));
|
|
82
|
-
const mass = zero();
|
|
83
|
-
for (const b of branches) b.reach.mass.forEach((m, i) => mass[i] += b.weight / total * m);
|
|
84
|
-
out = { mass, known: true };
|
|
85
|
-
} else if (live.length === 0) out = { mass: zero(), known: true };
|
|
86
|
-
else if (live.every((b) => sameShares(b.reach.mass, live[0].reach.mass))) {
|
|
87
|
-
const ref = live[0].reach.mass;
|
|
88
|
-
out = { mass: ref.map((m) => m / sum(ref)), known: false };
|
|
89
|
-
} else out = "unknown";
|
|
90
|
-
}
|
|
91
|
-
memo.set(id, out);
|
|
92
|
-
return out;
|
|
93
|
-
};
|
|
94
|
-
const reach = walk(flow.start, 0);
|
|
95
|
-
if (reach === "unknown" || sum(reach.mass) <= 0) return null;
|
|
96
|
-
return reach.mass;
|
|
97
|
-
}
|
|
98
12
|
function resultsExperiment(config, key) {
|
|
99
13
|
return getExperiment(config, key) ?? (key === HOLDOUT_KEY ? holdoutExperiment(config) : null);
|
|
100
14
|
}
|
|
101
|
-
function
|
|
15
|
+
function rawVariants(experiment, observed, withRevenue) {
|
|
102
16
|
return experiment.variants.map((v) => {
|
|
103
17
|
const row = observed.find((o) => o.variant === v.key);
|
|
18
|
+
const visitors = row?.exposures ?? 0;
|
|
19
|
+
const conversions = row?.conversions ?? 0;
|
|
20
|
+
const revenue = row?.revenueSum ?? row?.revenue;
|
|
104
21
|
return {
|
|
105
22
|
variant: v.key,
|
|
106
|
-
|
|
107
|
-
conversions
|
|
108
|
-
|
|
109
|
-
...
|
|
110
|
-
...row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
|
|
23
|
+
visitors,
|
|
24
|
+
conversions,
|
|
25
|
+
rate: visitors > 0 ? conversions / visitors : 0,
|
|
26
|
+
...withRevenue && revenue !== void 0 ? { revenue } : {},
|
|
27
|
+
...withRevenue && row?.revenueSumSq !== void 0 ? { revenueSumSq: row.revenueSumSq } : {}
|
|
111
28
|
};
|
|
112
29
|
});
|
|
113
30
|
}
|
|
114
|
-
async function
|
|
115
|
-
const m = experiment.metrics;
|
|
116
|
-
if (!m) return [];
|
|
117
|
-
const list = [
|
|
118
|
-
...m.secondary.map((metric) => ({ metric, role: "secondary" })),
|
|
119
|
-
...m.guardrails.map((g) => ({ metric: g.goal, role: "guardrail", worse: g.worse }))
|
|
120
|
-
];
|
|
121
|
-
return Promise.all(list.map(async (x) => ({ ...x, counts: alignCounts(experiment, await store.variantCounts(experiment.key, x.metric)) })));
|
|
122
|
-
}
|
|
123
|
-
async function experimentResults(config, store, key, options = {}) {
|
|
31
|
+
async function rawResults(config, store, key, options = {}) {
|
|
124
32
|
const experiment = resultsExperiment(config, key);
|
|
125
33
|
if (!experiment) return null;
|
|
126
34
|
const goal2 = experiment.goal;
|
|
127
35
|
const observed = goal2 ? await store.variantCounts(key, goal2) : [];
|
|
128
|
-
const
|
|
129
|
-
const
|
|
130
|
-
const
|
|
131
|
-
const
|
|
132
|
-
const
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (experiment.flow) {
|
|
138
|
-
const shares = flowExpectedShares(experiment, experiment.flow);
|
|
139
|
-
expected = shares ? { expectedWeights: shares } : { skipSrm: "Split check skipped: targeting conditions decide which variant a visitor reaches, so the expected split depends on who visits." };
|
|
140
|
-
}
|
|
141
|
-
const isMvt = experiment.kind === "mvt" && Boolean(goal2);
|
|
142
|
-
const familyPValues = isMvt ? factorPValues(experiment, counts2) : void 0;
|
|
143
|
-
const verdict = goal2 ? computeVerdict(key, counts2, { ...expected, days, familyPValues, now, ...experiment.minDays !== void 0 ? { minDays: experiment.minDays } : {}, ...metrics.length ? { metrics } : {} }) : { ...computeVerdict(key, [], { now }), message: "No goal configured for this experiment; add `goal` in splithub.config.ts." };
|
|
144
|
-
const factors = isMvt ? factorEffects(experiment, counts2, { familyPValues: verdict.comparisons.map((c) => c.sequential.pValue) }) : void 0;
|
|
145
|
-
const revenue = goal2 ? revenueResult(counts2, Boolean(config.goals.find((g) => g.key === goal2)?.revenue)) : void 0;
|
|
36
|
+
const goalHasRevenue = Boolean(goal2 && config.goals.find((g) => g.key === goal2)?.revenue);
|
|
37
|
+
const withRevenue = goalHasRevenue || observed.some((o) => (o.revenueSum ?? o.revenue ?? 0) !== 0);
|
|
38
|
+
const summary = options.summary !== void 0 ? options.summary : (await store.summaries()).find((s) => s.experiment === key) ?? null;
|
|
39
|
+
const span = summary?.firstExposureAt != null && summary.lastExposureAt != null ? (summary.lastExposureAt - summary.firstExposureAt) / DAY_MS : 0;
|
|
40
|
+
const m = goal2 ? experiment.metrics : void 0;
|
|
41
|
+
const metrics = m ? await Promise.all([
|
|
42
|
+
...m.secondary.map(async (metric) => ({ metric, role: "secondary", variants: rawVariants(experiment, await store.variantCounts(key, metric), false) })),
|
|
43
|
+
...m.guardrails.map(async (g) => ({ metric: g.goal, role: "guardrail", ...g.worse ? { worse: g.worse } : {}, variants: rawVariants(experiment, await store.variantCounts(key, g.goal), false) }))
|
|
44
|
+
]) : void 0;
|
|
146
45
|
return {
|
|
147
46
|
experiment: key,
|
|
148
47
|
kind: experiment.kind,
|
|
149
48
|
status: experiment.status,
|
|
150
49
|
goal: goal2,
|
|
151
|
-
description: experiment.description,
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
...revenue ? { revenue } : {}
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
function revenueResult(counts2, goalHasRevenue) {
|
|
160
|
-
if (!counts2.every((c) => typeof c.revenueSumSq === "number")) return void 0;
|
|
161
|
-
const rows = counts2.map((c) => ({ variant: c.variant, n: c.exposures, sum: c.revenueSum ?? c.revenue ?? 0, sumSq: c.revenueSumSq ?? 0 }));
|
|
162
|
-
if (!goalHasRevenue && rows.every((r) => r.sum === 0)) return void 0;
|
|
163
|
-
return {
|
|
164
|
-
variants: rows.map((r) => ({ variant: r.variant, visitors: r.n, revenue: r.sum, perVisitor: r.n > 0 ? r.sum / r.n : 0 })),
|
|
165
|
-
comparisons: compareMeans(rows)
|
|
50
|
+
description: experiment.description ?? null,
|
|
51
|
+
variants: rawVariants(experiment, observed, withRevenue),
|
|
52
|
+
...metrics?.length ? { metrics } : {},
|
|
53
|
+
days: Math.round(span * 10) / 10,
|
|
54
|
+
analysis: ANALYSIS_UNAVAILABLE
|
|
166
55
|
};
|
|
167
56
|
}
|
|
168
|
-
async function
|
|
169
|
-
const
|
|
170
|
-
const byKey = new Map(summaries.map((s) => [s.experiment, s]));
|
|
171
|
-
const rows = [];
|
|
57
|
+
async function rawStatusTable(config, store) {
|
|
58
|
+
const byKey = new Map((await store.summaries()).map((s) => [s.experiment, s]));
|
|
172
59
|
const holdout = holdoutExperiment(config);
|
|
173
60
|
const all = holdout && byKey.has(HOLDOUT_KEY) ? [...config.experiments, holdout] : config.experiments;
|
|
174
|
-
|
|
61
|
+
return all.map((exp) => {
|
|
175
62
|
const s = byKey.get(exp.key);
|
|
176
|
-
const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) /
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
let best = null;
|
|
180
|
-
if (exp.goal && s) {
|
|
181
|
-
const r = await experimentResults(config, store, exp.key, { summary: s });
|
|
182
|
-
verdict = r?.verdict.status ?? null;
|
|
183
|
-
headline = r ? headlineFor(r.verdict) : null;
|
|
184
|
-
best = r ? bestOf(r.verdict) : null;
|
|
185
|
-
}
|
|
186
|
-
rows.push({ experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days, verdict, headline, best });
|
|
187
|
-
}
|
|
188
|
-
return rows;
|
|
189
|
-
}
|
|
190
|
-
function bestOf(v) {
|
|
191
|
-
const best = v.comparisons.slice().sort((a, b) => b.absoluteLift - a.absoluteLift)[0];
|
|
192
|
-
const name = v.winner ?? best?.variant;
|
|
193
|
-
if (!name) return null;
|
|
194
|
-
const cmp = best && best.variant === name ? best : v.comparisons.find((c) => c.variant === name);
|
|
195
|
-
return { variant: name, index: v.sample.variants.findIndex((x) => x.variant === name) + 1, lift: cmp?.relativeLift ?? null };
|
|
196
|
-
}
|
|
197
|
-
function headlineFor(v) {
|
|
198
|
-
const best = bestOf(v);
|
|
199
|
-
const lift = best?.lift != null ? `${best.lift >= 0 ? "+" : ""}${(best.lift * 100).toFixed(1)}%` : "";
|
|
200
|
-
switch (v.status) {
|
|
201
|
-
case "winner_found":
|
|
202
|
-
return `#${best?.index ?? "?"} wins ${lift}`.trim();
|
|
203
|
-
case "insufficient_sample":
|
|
204
|
-
return best ? `#${best.index} ${lift} (too early)` : "too early";
|
|
205
|
-
case "srm_detected":
|
|
206
|
-
return "SRM! check setup";
|
|
207
|
-
case "guardrail_breached":
|
|
208
|
-
return `guardrail hit: ${v.guardrailBreached?.[0]?.metric ?? "?"}`;
|
|
209
|
-
case "no_significant_difference":
|
|
210
|
-
return "no difference";
|
|
211
|
-
default:
|
|
212
|
-
return "no data";
|
|
213
|
-
}
|
|
63
|
+
const days = s?.firstExposureAt && s.lastExposureAt ? Math.max(1, Math.ceil((s.lastExposureAt - s.firstExposureAt) / DAY_MS)) : 0;
|
|
64
|
+
return { experiment: exp.key, kind: exp.kind, status: exp.status, goal: exp.goal, description: exp.description ?? null, exposures: s?.exposures ?? 0, visitors: s?.visitors ?? 0, days };
|
|
65
|
+
});
|
|
214
66
|
}
|
|
215
67
|
|
|
216
68
|
// src/validate.ts
|
|
@@ -306,9 +158,9 @@ function createSplitHubHandler(options) {
|
|
|
306
158
|
if (request.method === "GET") {
|
|
307
159
|
if (tail[1] === "health") return json(request, { ok: true, project: config.project });
|
|
308
160
|
if (!authorized(request)) return json(request, { error: "unauthorized" }, 401);
|
|
309
|
-
if (tail[1] === "status") return json(request, { project: config.project, experiments: await
|
|
161
|
+
if (tail[1] === "status") return json(request, { project: config.project, experiments: await rawStatusTable(config, store), analysis: ANALYSIS_UNAVAILABLE });
|
|
310
162
|
if (tail[0] === "results" && tail[1]) {
|
|
311
|
-
const r = await
|
|
163
|
+
const r = await rawResults(config, store, tail[1]);
|
|
312
164
|
return r ? json(request, r) : json(request, { error: `unknown experiment "${tail[1]}"` }, 404);
|
|
313
165
|
}
|
|
314
166
|
}
|
|
@@ -627,22 +479,19 @@ function sortCounts(rows) {
|
|
|
627
479
|
}).sort((a, b) => a.variant < b.variant ? -1 : a.variant > b.variant ? 1 : 0);
|
|
628
480
|
}
|
|
629
481
|
export {
|
|
482
|
+
ANALYSIS_UNAVAILABLE,
|
|
630
483
|
ATTRIBUTION_SKEW_MS,
|
|
631
484
|
GOAL_DAILY_CAP,
|
|
632
485
|
attributeVariants,
|
|
633
486
|
attributionConformance,
|
|
634
|
-
bestOf,
|
|
635
487
|
countsFromRow,
|
|
636
488
|
createMemoryStore,
|
|
637
489
|
createSplitHubHandler,
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
flowExpectedShares,
|
|
641
|
-
headlineFor,
|
|
490
|
+
rawResults,
|
|
491
|
+
rawStatusTable,
|
|
642
492
|
resultsExperiment,
|
|
643
493
|
revenueOf,
|
|
644
494
|
sortCounts,
|
|
645
|
-
statusTable,
|
|
646
495
|
toNodeListener,
|
|
647
496
|
validateBatch,
|
|
648
497
|
variantCountsSql
|