@splithub/node 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1,7 +1,62 @@
1
- import { SplitHubConfig, Experiment } from '@splithub/core';
2
- import { VariantCounts, Verdict } from '@splithub/stats';
1
+ import { SplitHubConfig, Experiment, Flow } from '@splithub/core';
2
+ import { VariantCounts, Verdict, MeanComparison } from '@splithub/stats';
3
3
  import { IncomingMessage, ServerResponse } from 'node:http';
4
4
 
5
+ /** How far a goal may precede the exposure it converts, for clock differences between server and browser. */
6
+ declare const ATTRIBUTION_SKEW_MS = 60000;
7
+ /** Events of one goal from one visitor counted per UTC day. */
8
+ declare const GOAL_DAILY_CAP = 50;
9
+ /** Per-variant counts every store returns: `revenueSum` / `revenueSumSq` are over per-visitor revenue (non-converted visitors add 0). */
10
+ interface AttributedCounts extends VariantCounts {
11
+ revenue: number;
12
+ revenueSum: number;
13
+ revenueSumSq: number;
14
+ }
15
+ /** The shape of an event the rule needs. */
16
+ interface AttributionEvent {
17
+ id: string;
18
+ type: 'exposure' | 'goal';
19
+ experiment: string | null;
20
+ variant: string | null;
21
+ goal: string | null;
22
+ visitorId: string;
23
+ ts: number;
24
+ props: Record<string, unknown> | null;
25
+ }
26
+ declare function revenueOf(props: Record<string, unknown> | null | undefined): number;
27
+ /** Reference implementation of the rule over events in memory. */
28
+ declare function attributeVariants(events: Iterable<AttributionEvent>, experiment: string, goal: string): AttributedCounts[];
29
+ type SqlDialect = 'sqlite' | 'postgres' | 'mysql';
30
+ type AttributionParam = 'experiment' | 'goal' | 'project';
31
+ interface VariantCountsSqlOptions {
32
+ dialect: SqlDialect;
33
+ /** Events table. Defaults to "splithub_events". */
34
+ table?: string;
35
+ /** Column partitioning a shared table by project (adapter-d1); bound from the `project` param. */
36
+ projectColumn?: string;
37
+ }
38
+ interface VariantCountsSql {
39
+ sql: string;
40
+ /** Names of the values to bind, in placeholder order (Postgres numbers them, so each name appears once). */
41
+ params: AttributionParam[];
42
+ /** Binds values in the right order. */
43
+ bind(values: {
44
+ experiment: string;
45
+ goal: string;
46
+ project?: string;
47
+ }): unknown[];
48
+ }
49
+ /**
50
+ * The attribution query for an events table with the columns of the SplitHub
51
+ * stores (id, type, experiment, variant, goal, visitor_id, ts, props). Returns
52
+ * one row per variant: variant, exposures, conversions, revenue_sum,
53
+ * revenue_sum_sq. Derived tables only, no CTEs or window functions, so the same
54
+ * shape runs on SQLite, D1, Postgres and MySQL 5.7.
55
+ */
56
+ declare function variantCountsSql(options: VariantCountsSqlOptions): VariantCountsSql;
57
+ /** Turns a row of `variantCountsSql` (numbers may come back as strings from pg / mysql2) into counts. */
58
+ declare function countsFromRow(row: Record<string, unknown>): AttributedCounts;
59
+
5
60
  /** One row in the events table, whatever the backing database. */
6
61
  interface StoredEvent {
7
62
  id: string;
@@ -41,21 +96,19 @@ interface ExperimentSummary {
41
96
  firstExposureAt: number | null;
42
97
  lastExposureAt: number | null;
43
98
  }
99
+ /** What `variantCounts` returns. Older custom stores may omit the revenue fields; `revenue` then stands for the sum. */
100
+ type StoreVariantCounts = Omit<AttributedCounts, 'revenue' | 'revenueSum' | 'revenueSumSq'> & Partial<Pick<AttributedCounts, 'revenue' | 'revenueSum' | 'revenueSumSq'>>;
44
101
  /**
45
102
  * Storage adapter. Implement this against your database to keep all data in
46
- * your own infrastructure. Attribution rule for `variantCounts`: a visitor
47
- * counts once per experiment, in the variant of their earliest exposure, and
48
- * converts when a goal arrived at or after that exposure.
103
+ * your own infrastructure. `variantCounts` follows the rule in `./attribution`
104
+ * (first exposure decides the variant, goals count from 60 s before it, at most
105
+ * 50 of one goal per visitor and day); SQL databases can run `variantCountsSql`.
49
106
  */
50
107
  interface Store {
51
108
  /** Insert events, ignoring ids already present. Returns the number inserted. */
52
109
  insertEvents(events: StoredEvent[]): Promise<number>;
53
- /**
54
- * Unique exposed and converted visitors per variant (any order). A visitor
55
- * belongs to the variant of their earliest exposure (same ts → smallest
56
- * variant key) and converts when a goal has ts >= that exposure.
57
- */
58
- variantCounts(experiment: string, goal: string): Promise<VariantCounts[]>;
110
+ /** Unique exposed and converted visitors per variant (any order), with per-visitor revenue sum and sum of squares. */
111
+ variantCounts(experiment: string, goal: string): Promise<StoreVariantCounts[]>;
59
112
  summaries(): Promise<ExperimentSummary[]>;
60
113
  close?(): Promise<void> | void;
61
114
  }
@@ -105,7 +158,13 @@ interface FactorLevelResult {
105
158
  rate: number;
106
159
  /** Relative lift vs the factor's first level; null for that level itself. */
107
160
  relativeLift: number | null;
161
+ /** Fixed-horizon p-value, for reference; null for the first level. */
108
162
  pValue: number | null;
163
+ /** Always-valid p-value (what `significant` is decided on); null for the first level. */
164
+ sequentialPValue: number | null;
165
+ /** Probability this level beats the first one, same formula as the per-combination table; null for the first level. */
166
+ probabilityToBeat: number | null;
167
+ /** Always-valid, Holm-corrected across every factor level and combination of the experiment. */
109
168
  significant: boolean;
110
169
  }
111
170
  /** A multivariate factor read on its own (its main effect), independent of the other factors. */
@@ -115,6 +174,17 @@ interface FactorResult {
115
174
  /** The level that beats the first one significantly with the largest lift, if any. */
116
175
  best: string | null;
117
176
  }
177
+ /** Revenue per visitor (visitors who did not pay count as 0), compared with the control by Welch's t-test. */
178
+ interface RevenueResult {
179
+ variants: Array<{
180
+ variant: string;
181
+ visitors: number;
182
+ revenue: number;
183
+ perVisitor: number;
184
+ }>;
185
+ /** Fixed-horizon and always-valid numbers per variant; `significant` uses the always-valid p, Holm-corrected. */
186
+ comparisons: MeanComparison[];
187
+ }
118
188
  interface ExperimentResults {
119
189
  experiment: string;
120
190
  kind: string;
@@ -122,16 +192,42 @@ interface ExperimentResults {
122
192
  goal: string | null;
123
193
  description: string | null;
124
194
  verdict: Verdict;
195
+ /** Days until a verdict at the current traffic: 0 once decided, null when there is no estimate. */
196
+ daysToVerdict: number | null;
197
+ /** Visitors per day behind `daysToVerdict`, null when unknown. */
198
+ visitorsPerDay: number | null;
125
199
  /** Multivariate experiments only: each factor's main effect. */
126
200
  factors?: FactorResult[];
201
+ /** Present when the store reports `revenueSum` and `revenueSumSq` per variant. */
202
+ revenue?: RevenueResult;
203
+ }
204
+ interface ResultsOptions {
205
+ now?: Date;
206
+ /** Days of data; defaults to the span between the first and last exposure in `store.summaries()`. */
207
+ days?: number;
208
+ /** Visitors per day for `daysToVerdict`; defaults to visitors / days. */
209
+ visitorsPerDay?: number;
210
+ /** The experiment's summary when the caller already has it (saves a `summaries()` call). */
211
+ summary?: ExperimentSummary | null;
127
212
  }
128
213
  /**
129
214
  * Main effects of a multivariate experiment: for each factor, every level's
130
215
  * visitors and conversions summed over the combinations that use it, compared
131
- * with the factor's first level. The same z-test as the verdict, so the
132
- * numbers agree with the per-combination table.
216
+ * with the factor's first level. Same tests as the verdict. Significance is
217
+ * Holm-corrected across all factor levels and, when `familyPValues` carries the
218
+ * combinations' always-valid p-values, across those too, so reading factors and
219
+ * combinations together still keeps the 5% false positive rate.
220
+ */
221
+ declare function factorEffects(experiment: Experiment, counts: VariantCounts[], options?: {
222
+ familyPValues?: number[];
223
+ }): FactorResult[];
224
+ /**
225
+ * Expected traffic shares of the variants under a targeting flow. Splits mix
226
+ * their outputs by weight. A condition sends an unknown share of visitors each
227
+ * way, so the split is only defined when every branch that reaches a variant
228
+ * leads to the same proportions. Null when it depends on who visits.
133
229
  */
134
- declare function factorEffects(experiment: Experiment, counts: VariantCounts[]): FactorResult[];
230
+ declare function flowExpectedShares(experiment: Experiment, flow: Flow): number[] | null;
135
231
  interface StatusRow {
136
232
  experiment: string;
137
233
  kind: string;
@@ -151,7 +247,9 @@ interface StatusRow {
151
247
  lift: number | null;
152
248
  } | null;
153
249
  }
154
- declare function experimentResults(config: SplitHubConfig, store: Store, key: string): Promise<ExperimentResults | null>;
250
+ /** An experiment of the config, or the holdout comparison for `__holdout`. */
251
+ declare function resultsExperiment(config: SplitHubConfig, key: string): Experiment | null;
252
+ declare function experimentResults(config: SplitHubConfig, store: Store, key: string, options?: ResultsOptions): Promise<ExperimentResults | null>;
155
253
  declare function statusTable(config: SplitHubConfig, store: Store): Promise<StatusRow[]>;
156
254
  /** The leading variant of a verdict with its ordinal (#1 is the control, in declared order). */
157
255
  declare function bestOf(v: Verdict): StatusRow['best'];
@@ -167,4 +265,21 @@ type ValidationResult = {
167
265
  /** `expectedProject` null skips the name check (the caller already identified the project, e.g. by public key). */
168
266
  declare function validateBatch(body: unknown, expectedProject: string | null, now?: number): ValidationResult;
169
267
 
170
- export { type ExperimentResults, type ExperimentSummary, type FactorLevelResult, type FactorResult, type FetchHandler, type HandlerOptions, type StatusRow, type Store, type StoredEvent, bestOf, createMemoryStore, createSplitHubHandler, experimentResults, factorEffects, headlineFor, statusTable, toNodeListener, validateBatch };
268
+ interface ConformanceCase {
269
+ name: string;
270
+ experiment: string;
271
+ goal: string;
272
+ events: StoredEvent[];
273
+ expected: AttributedCounts[];
274
+ /**
275
+ * The cloud decides at ingest in arrival order, so a later-arriving exposure
276
+ * with an earlier ts cannot move a visitor. Cases that depend on that are
277
+ * only for the read-time stores.
278
+ */
279
+ readTimeOnly?: boolean;
280
+ }
281
+ declare const attributionConformance: ConformanceCase[];
282
+ /** Sorts by variant and keeps the compared fields, so any store's output can be `toEqual`-ed with `expected`. */
283
+ declare function sortCounts(rows: StoreVariantCounts[]): AttributedCounts[];
284
+
285
+ export { ATTRIBUTION_SKEW_MS, type AttributedCounts, type AttributionEvent, type ConformanceCase, type ExperimentResults, type ExperimentSummary, type FactorLevelResult, type FactorResult, type FetchHandler, GOAL_DAILY_CAP, type HandlerOptions, type ResultsOptions, type RevenueResult, type SqlDialect, type StatusRow, type Store, type StoreVariantCounts, type StoredEvent, type VariantCountsSql, type VariantCountsSqlOptions, attributeVariants, attributionConformance, bestOf, countsFromRow, createMemoryStore, createSplitHubHandler, experimentResults, factorEffects, flowExpectedShares, headlineFor, resultsExperiment, revenueOf, sortCounts, statusTable, toNodeListener, validateBatch, variantCountsSql };