@ak--47/dungeon-master 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +96 -0
- package/.claude/skills/create-project/provision.mjs +336 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +211 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +3 -2
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -22,8 +22,12 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import { toMs } from '../hook-helpers/_internal.js';
|
|
25
|
-
import { evaluateFunnel, evaluateAnyOrderCompletion } from './funnel-engine.js';
|
|
25
|
+
import { evaluateFunnel, evaluateFunnelHPC, evaluateAnyOrderCompletion } from './funnel-engine.js';
|
|
26
26
|
import { buildIdentityMap, resolveUserId } from './identity.js';
|
|
27
|
+
import { coerceToBreakdownKey, breakdownSegmentKey, matchesWhere } from './coerce.js';
|
|
28
|
+
import { filterFirstTimeEver } from './first-time.js';
|
|
29
|
+
import { sessionize } from './sessionize.js';
|
|
30
|
+
import { extractFlows, aggregateFlows } from './flows.js';
|
|
27
31
|
import {
|
|
28
32
|
countDistinctPeriods,
|
|
29
33
|
countDistinctValues,
|
|
@@ -31,69 +35,80 @@ import {
|
|
|
31
35
|
nullAwareSum,
|
|
32
36
|
nullAwareExtreme,
|
|
33
37
|
partitionByTimeBucket,
|
|
38
|
+
bucketBoundsMs,
|
|
34
39
|
} from './counting.js';
|
|
35
40
|
|
|
36
41
|
/**
|
|
37
42
|
* v1.5: pick the right per-user funnel evaluator based on the funnel's `order` mode.
|
|
43
|
+
* v1.6.0 (P1.6.6): orders whose scrambled region is CONTIGUOUS map exactly onto
|
|
44
|
+
* the engine's `{ anyOrder: [...] }` step blocks (history.cpp anchor/chunk
|
|
45
|
+
* greedy pass) — full Mixpanel semantics: conversion window, 2s rule,
|
|
46
|
+
* exclusions, reentry, and `anchorRange` all apply.
|
|
38
47
|
*
|
|
39
|
-
* - `sequential`, `interrupt`, `interrupted` → greedy single-pass
|
|
40
|
-
* - `first-fixed`
|
|
41
|
-
* - `last-fixed
|
|
42
|
-
*
|
|
43
|
-
* - `random`
|
|
48
|
+
* - `sequential`, `interrupt`, `interrupted` → greedy single-pass
|
|
49
|
+
* - `first-fixed` → `[s0, { anyOrder: rest }]`
|
|
50
|
+
* - `last-fixed` → `[{ anyOrder: init }, sLast]`
|
|
51
|
+
* - `first-and-last-fixed` → `[s0, { anyOrder: middle }, sLast]`
|
|
52
|
+
* - `outside-in`, `random` → `[{ anyOrder: all }]` (full any-order)
|
|
53
|
+
* - `middle-fixed` → set-membership completion (partial): the
|
|
54
|
+
* scrambled slots are the two ENDS (`u.shuffleOutside` — middle stays
|
|
55
|
+
* fixed), which is non-contiguous; neither the engine nor Mixpanel's
|
|
56
|
+
* any-order groups can express it.
|
|
44
57
|
*
|
|
45
58
|
* Returns a normalized result `{ completed, reached, ttcMs, mode, verificationKind }`.
|
|
46
|
-
* `verificationKind` is `undefined` for
|
|
47
|
-
* `'partial'` for
|
|
59
|
+
* `verificationKind` is `undefined` for Mixpanel-comparable modes (full
|
|
60
|
+
* PASS/FAIL allowed), `'partial'` for `middle-fixed`'s completion-only check.
|
|
48
61
|
*
|
|
49
62
|
* @param {Array<Object>} userEvents
|
|
50
63
|
* @param {string[]} steps
|
|
51
64
|
* @param {Object} options
|
|
52
65
|
* @param {string} [options.funnelOrder='sequential']
|
|
53
66
|
* @param {number} [options.conversionWindowMs]
|
|
67
|
+
* @param {{unit: 'sessions', n: number}} [options.conversionWindow]
|
|
54
68
|
* @param {boolean} [options.reentry]
|
|
55
69
|
* @param {Array<Object>} [options.exclusionSteps]
|
|
56
70
|
* @param {boolean | string[]} [options.trackStepProperties]
|
|
57
71
|
* @param {boolean} [options.sessionScoped]
|
|
72
|
+
* @param {{fromMs?: number, toMs?: number}} [options.anchorRange] - step-0
|
|
73
|
+
* anchor bounds (P1.6.5) — honored by every order except `middle-fixed`
|
|
74
|
+
* (whose set-membership check has no anchor concept).
|
|
58
75
|
*/
|
|
59
76
|
function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
60
77
|
const order = options.funnelOrder || 'sequential';
|
|
61
78
|
const sequentialOpts = {
|
|
62
79
|
conversionWindowMs: options.conversionWindowMs,
|
|
80
|
+
conversionWindow: options.conversionWindow,
|
|
63
81
|
reentry: options.reentry,
|
|
64
82
|
exclusionSteps: options.exclusionSteps,
|
|
65
83
|
trackStepProperties: options.trackStepProperties,
|
|
66
84
|
sessionScoped: options.sessionScoped,
|
|
85
|
+
anchorRange: options.anchorRange,
|
|
67
86
|
};
|
|
87
|
+
// Degenerate scrambled regions collapse to plain steps: an empty region
|
|
88
|
+
// means the order IS sequential, and a 1-element region shuffles to itself
|
|
89
|
+
// (the generator's shuffle of one element is the identity) — expressing it
|
|
90
|
+
// as a 1-member anyOrder block would change buffering semantics (chunk
|
|
91
|
+
// first-match vs anchor latest-match) for data that is in fact ordered.
|
|
92
|
+
const block = (arr) => arr.length === 1 ? arr[0] : { anyOrder: arr };
|
|
93
|
+
let engineSteps = /** @type {Array<*>} */ (steps);
|
|
68
94
|
switch (order) {
|
|
69
|
-
case '
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const r = /** @type {*} */ (evaluateFunnel(userEvents, steps, sequentialOpts));
|
|
73
|
-
return { ...r, mode: order, verificationKind: undefined };
|
|
74
|
-
}
|
|
75
|
-
case 'first-fixed': {
|
|
76
|
-
const stepZero = /** @type {*} */ (evaluateFunnel(userEvents, [steps[0]], { conversionWindowMs: options.conversionWindowMs }));
|
|
77
|
-
if (!stepZero.completed) {
|
|
78
|
-
return { completed: false, reached: -1, stepEvents: [], stepTimes: [], ttcMs: null, mode: 'first-fixed', verificationKind: 'partial' };
|
|
79
|
-
}
|
|
80
|
-
const rest = evaluateAnyOrderCompletion(userEvents, steps.slice(1));
|
|
81
|
-
const completed = rest.completed;
|
|
82
|
-
const reached = completed ? steps.length - 1 : 0;
|
|
83
|
-
return {
|
|
84
|
-
completed,
|
|
85
|
-
reached,
|
|
86
|
-
stepEvents: [],
|
|
87
|
-
stepTimes: [],
|
|
88
|
-
ttcMs: completed ? rest.completionTimeMs : null,
|
|
89
|
-
mode: 'first-fixed',
|
|
90
|
-
verificationKind: 'partial',
|
|
91
|
-
};
|
|
92
|
-
}
|
|
95
|
+
case 'first-fixed':
|
|
96
|
+
if (steps.length >= 2) engineSteps = [steps[0], block(steps.slice(1))];
|
|
97
|
+
break;
|
|
93
98
|
case 'last-fixed':
|
|
94
|
-
|
|
99
|
+
if (steps.length >= 2) engineSteps = [block(steps.slice(0, -1)), steps[steps.length - 1]];
|
|
100
|
+
break;
|
|
95
101
|
case 'first-and-last-fixed':
|
|
96
|
-
|
|
102
|
+
if (steps.length >= 3) engineSteps = [steps[0], block(steps.slice(1, -1)), steps[steps.length - 1]];
|
|
103
|
+
break;
|
|
104
|
+
case 'outside-in':
|
|
105
|
+
case 'random':
|
|
106
|
+
// Full any-order group. `random` generation is a full shuffle;
|
|
107
|
+
// `outside-in` has no generator implementation (falls through to
|
|
108
|
+
// sequential order) — the any-order funnel accepts both.
|
|
109
|
+
if (steps.length >= 2) engineSteps = [{ anyOrder: steps.slice() }];
|
|
110
|
+
break;
|
|
111
|
+
case 'middle-fixed': {
|
|
97
112
|
const r = evaluateAnyOrderCompletion(userEvents, steps);
|
|
98
113
|
return {
|
|
99
114
|
completed: r.completed,
|
|
@@ -105,28 +120,16 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
105
120
|
verificationKind: 'partial',
|
|
106
121
|
};
|
|
107
122
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
return {
|
|
111
|
-
completed: r.completed,
|
|
112
|
-
reached: r.completed ? steps.length - 1 : -1,
|
|
113
|
-
stepEvents: [],
|
|
114
|
-
stepTimes: [],
|
|
115
|
-
ttcMs: r.completionTimeMs,
|
|
116
|
-
mode: 'random',
|
|
117
|
-
verificationKind: 'informational',
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
default: {
|
|
121
|
-
const r = /** @type {*} */ (evaluateFunnel(userEvents, steps, sequentialOpts));
|
|
122
|
-
return { ...r, mode: order, verificationKind: undefined };
|
|
123
|
-
}
|
|
123
|
+
default:
|
|
124
|
+
break; // sequential / interrupt / interrupted / unknown → as-is
|
|
124
125
|
}
|
|
126
|
+
const r = /** @type {*} */ (evaluateFunnel(userEvents, engineSteps, sequentialOpts));
|
|
127
|
+
return { ...r, mode: order, verificationKind: undefined };
|
|
125
128
|
}
|
|
126
129
|
|
|
127
130
|
/**
|
|
128
131
|
* @typedef {Object} EmulateOptions
|
|
129
|
-
* @property {'frequencyByFrequency'|'funnelFrequency'|'aggregatePerUser'|'timeToConvert'|'attributedBy'|'sessionMetrics'|'retention'|'distinctCount'} type
|
|
132
|
+
* @property {'frequencyByFrequency'|'funnelFrequency'|'aggregatePerUser'|'timeToConvert'|'attributedBy'|'sessionMetrics'|'retention'|'distinctCount'|'eventBreakdown'|'uniques'|'lifecycle'|'topPaths'} type
|
|
130
133
|
*
|
|
131
134
|
* @property {string} [metricEvent]
|
|
132
135
|
* @property {string} [breakdownByFrequencyOf]
|
|
@@ -141,7 +144,10 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
141
144
|
* @property {'avg'|'sum'|'count'|'max'|'min'} [agg]
|
|
142
145
|
*
|
|
143
146
|
* @property {string} [fromEvent]
|
|
144
|
-
* @property {string} [toEvent]
|
|
147
|
+
* @property {string} [toEvent] - timeToConvert pair sugar. v1.6.0 (P1.6.7):
|
|
148
|
+
* pass `steps` (>= 2 entries) instead for multi-step TTC; rows gain
|
|
149
|
+
* avg_ttc_s, step_counts, gap_avg_s, cumulative_avg_s
|
|
150
|
+
* (funnel_query.cpp:3355-3380 arithmetic — integer seconds, per-gap clamp).
|
|
145
151
|
* @property {string} [breakdownByUserProperty]
|
|
146
152
|
* @property {Array<Object>} [profiles]
|
|
147
153
|
*
|
|
@@ -149,7 +155,61 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
149
155
|
* @property {string} [attributionEvent]
|
|
150
156
|
* @property {string} [attributionProperty]
|
|
151
157
|
* @property {'firstTouch'|'lastTouch'} [model]
|
|
152
|
-
* @property {
|
|
158
|
+
* @property {'first'|'all'} [perConversion] - attributedBy only: 'first' (default) evaluates
|
|
159
|
+
* only each user's first conversion (v1.5 behavior); 'all' runs attribution per conversion
|
|
160
|
+
* event (Mixpanel behavior — attributed_value_reader_read takes one
|
|
161
|
+
* event_time_ms per conversion).
|
|
162
|
+
*
|
|
163
|
+
* v1.6.0 eventBreakdown (Insights: total events segmented by a property):
|
|
164
|
+
* @property {string} [breakdownProperty]
|
|
165
|
+
* @property {number} [topN]
|
|
166
|
+
* @property {'sessions'|'unique'|'general'} [countType] - eventBreakdown +
|
|
167
|
+
* uniques (P1.7.3) accept 'sessions' only: count once per (user, session,
|
|
168
|
+
* segment) / distinct (user, session) pairs per bucket, deriving sessions
|
|
169
|
+
* via sessionize(). topPaths (P1.9) accepts all three (see extractFlows).
|
|
170
|
+
* @property {number} [sessionTimeoutMs]
|
|
171
|
+
*
|
|
172
|
+
* v1.6.0 (P1.7.2) sessionMetrics: `source: 'derived'` (default) re-derives
|
|
173
|
+
* sessions at query time via sessionize(); `'stamped'` keeps the v1.5
|
|
174
|
+
* stamped-session_id path. `sessionTimeoutMs`/`maxSessionMs` thread into
|
|
175
|
+
* sessionize. Every row reports `source` + `stampedDivergence`.
|
|
176
|
+
* @property {('count'|'duration'|'eventsPerSession')[]} [metrics]
|
|
177
|
+
* @property {'derived'|'stamped'} [source]
|
|
178
|
+
* @property {number} [maxSessionMs]
|
|
179
|
+
*
|
|
180
|
+
* v1.6.0 uniques (Insights: unique users per interval / XAU / cumulative):
|
|
181
|
+
* @property {Object<string, *>} [where]
|
|
182
|
+
* @property {('day'|'week'|'month'|'range')} [unit]
|
|
183
|
+
* @property {number} [rollingWindow]
|
|
184
|
+
* @property {boolean} [cumulative]
|
|
185
|
+
* @property {boolean} [firstTimeOnly]
|
|
186
|
+
*
|
|
187
|
+
* v1.6.0 lifecycle (P1.8 — the Lifecycle Cohort Analysis classification;
|
|
188
|
+
* New / Retained / Resurrected / Dormant user counts per tiled period):
|
|
189
|
+
* @property {string} [valueMomentEvent] - The event that defines "active".
|
|
190
|
+
* @property {7|30} [periodDays] - The two LCA template variants.
|
|
191
|
+
*
|
|
192
|
+
* v1.6.0 topPaths (P1.9 — the Flows report; anchored event paths aggregated
|
|
193
|
+
* into a pruned prefix tree; see flows.js for the ARB citations):
|
|
194
|
+
* @property {Array<string|{event: string, where?: Object<string, *>}>} [anchors]
|
|
195
|
+
* @property {number|number[]} [forward] - Steps kept AFTER each anchor
|
|
196
|
+
* (linear buffer, keeps the FIRST N). Scalar broadcasts; default 4 mirrors
|
|
197
|
+
* the UI's expansion default (spec contract, not an ARB constant).
|
|
198
|
+
* @property {number|number[]} [reverse] - Steps kept BEFORE each anchor
|
|
199
|
+
* (ring buffer, keeps the LAST N). Scalar broadcasts; default 0.
|
|
200
|
+
* @property {'list'|'sankey'} [output] - Result shape (default 'list').
|
|
201
|
+
* @property {number} [cardinalityThreshold] - Per-level top-N before
|
|
202
|
+
* coalescing into $mp_uncommon_flows_events (default 50 list / 3 sankey —
|
|
203
|
+
* bookmark.py:96/:110). In list mode the same value caps the returned
|
|
204
|
+
* rows (flows_merger.cpp:406-410; 0 = no row cap).
|
|
205
|
+
* @property {'end'|'start'} [listSortPriority] - List row ordering, by leaf
|
|
206
|
+
* count ('end', default) or slot-0 count ('start') — bookmark.py:95.
|
|
207
|
+
* @property {string[]} [hiddenEvents] - Non-anchor steps to drop (anchor
|
|
208
|
+
* names exempt — query/flows.go filterAnchorEventNames).
|
|
209
|
+
* @property {string[]} [visibleEvents] - Non-anchor allow-list; hidden wins
|
|
210
|
+
* (query/flows.go filteredVisibleEventSelectors).
|
|
211
|
+
* @property {boolean} [collapseRepeated] - Suppress consecutive repeats
|
|
212
|
+
* (default false — flows_params_utils.py:37).
|
|
153
213
|
*
|
|
154
214
|
* v1.5.0 funnel extensions (apply to funnelFrequency + timeToConvert sequential modes):
|
|
155
215
|
* @property {boolean} [reentry]
|
|
@@ -157,6 +217,23 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
157
217
|
* @property {boolean | string[]} [trackStepProperties]
|
|
158
218
|
* @property {boolean} [sessionScoped]
|
|
159
219
|
*
|
|
220
|
+
* v1.6.0 funnel count/window extensions (funnelFrequency, sequential order modes
|
|
221
|
+
* only — findings #15). `countMode` follows evaluateFunnel: 'uniques' (default,
|
|
222
|
+
* one result per user), 'totals' (every attempt counts a conversion row),
|
|
223
|
+
* 'sessions' (Mixpanel's session count type — rewritten to totals +
|
|
224
|
+
* 1-session window per api/version_2_0/arb_funnels/validate.py
|
|
225
|
+
* __validate_sessions). `holdPropertyConstant` splits each user into parallel
|
|
226
|
+
* per-property-value sub-funnels (evaluateFunnelHPC — scalar values only).
|
|
227
|
+
* `conversionWindow` is the session-count window ({ unit: 'sessions', n ≤ 12 });
|
|
228
|
+
* mutually exclusive with `conversionWindowMs`.
|
|
229
|
+
* @property {'uniques'|'totals'|'sessions'} [countMode]
|
|
230
|
+
* @property {string} [holdPropertyConstant]
|
|
231
|
+
* @property {{unit: 'sessions', n: number}} [conversionWindow]
|
|
232
|
+
* @property {string} [funnelOrder] - Funnel step-order mode (see
|
|
233
|
+
* evaluateFunnelByOrder); defaults to 'sequential'.
|
|
234
|
+
* @property {{from?: number|string, to?: number|string}} [cohortWindow] -
|
|
235
|
+
* Retention birth window (inclusive bounds — see retention()).
|
|
236
|
+
*
|
|
160
237
|
* @property {Map<string,string>} [identityMap]
|
|
161
238
|
*
|
|
162
239
|
* Cross-cutting time-bucketed output:
|
|
@@ -170,6 +247,21 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
170
247
|
* numerical aggregation. Mixpanel `normal_query.cpp:352-356` emits zero
|
|
171
248
|
* rows for empty intervals; we use the `_empty` marker instead of
|
|
172
249
|
* guessing a per-type zero-row template.
|
|
250
|
+
*
|
|
251
|
+
* v1.6.0 (P1.6.5) — step-0-anchored trends: under `timeBucket`, the types
|
|
252
|
+
* `funnelFrequency` and `timeToConvert` and `retention` follow Mixpanel's
|
|
253
|
+
* interval rule "step 0 in [start, stop); steps 1+ in
|
|
254
|
+
* [start, stop + conversion window)" (funnel_query.cpp:1398-1401;
|
|
255
|
+
* history.cpp:437-440 for retention returns): a conversion or return
|
|
256
|
+
* spilling past the bucket edge is credited to the bucket that ANCHORED it.
|
|
257
|
+
* P1.6.6 extended this to every funnel order that routes through the engine's
|
|
258
|
+
* anyOrder blocks (all orders except `middle-fixed`, whose set-membership
|
|
259
|
+
* check keeps plain event partitioning).
|
|
260
|
+
* @property {{fromMs?: number, toMs?: number}} [anchorRange] - Step-0 anchor
|
|
261
|
+
* bounds for `funnelFrequency`/`timeToConvert` (set automatically by the
|
|
262
|
+
* timeBucket wrapper; may be passed directly for a single-interval run).
|
|
263
|
+
* Events outside `[fromMs, toMs)` cannot anchor step 0 or count on the
|
|
264
|
+
* frequency axis, but can complete later steps.
|
|
173
265
|
*/
|
|
174
266
|
|
|
175
267
|
/**
|
|
@@ -178,7 +270,8 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
|
|
|
178
270
|
*
|
|
179
271
|
* @param {Array<Object>} events
|
|
180
272
|
* @param {EmulateOptions} config
|
|
181
|
-
* @returns {Array<Object
|
|
273
|
+
* @returns {Array<Object>|Object} Breakdown table rows (every type except
|
|
274
|
+
* 'topPaths', which returns aggregateFlows' single result object).
|
|
182
275
|
*/
|
|
183
276
|
export function emulateBreakdown(events, config) {
|
|
184
277
|
if (!Array.isArray(events)) throw new Error('emulateBreakdown: events must be an array');
|
|
@@ -207,13 +300,113 @@ export function emulateBreakdown(events, config) {
|
|
|
207
300
|
// emits zero rows for empty intervals). Consumers MUST filter `r._empty`
|
|
208
301
|
// before any aggregation.
|
|
209
302
|
if (config.timeBucket) {
|
|
303
|
+
if (config.type === 'uniques') {
|
|
304
|
+
// `uniques` produces its own time axis (unit/rollingWindow/cumulative).
|
|
305
|
+
// Pre-partitioning here would break rolling look-back windows and the
|
|
306
|
+
// cumulative running set, which both span bucket boundaries.
|
|
307
|
+
throw new Error('emulateBreakdown: type "uniques" does not compose with timeBucket — use `unit` instead');
|
|
308
|
+
}
|
|
309
|
+
if (config.type === 'lifecycle') {
|
|
310
|
+
// Classification is cross-period by construction (retained/resurrected/
|
|
311
|
+
// dormant all read T−1 and earlier) — pre-partitioning would blind it.
|
|
312
|
+
throw new Error('emulateBreakdown: type "lifecycle" does not compose with timeBucket — it tiles its own period axis');
|
|
313
|
+
}
|
|
314
|
+
if (config.type === 'topPaths') {
|
|
315
|
+
// Flows aggregates ONE path universe over the whole query range (a
|
|
316
|
+
// single prefix tree — flows_query.cpp end-of-data flush); there is
|
|
317
|
+
// no per-interval trend variant to partition into.
|
|
318
|
+
throw new Error('emulateBreakdown: type "topPaths" does not compose with timeBucket — flows aggregate one path universe over the range');
|
|
319
|
+
}
|
|
210
320
|
const range = config.timeBucketRange || {};
|
|
211
321
|
const buckets = partitionByTimeBucket(events, config.timeBucket, range);
|
|
212
322
|
// Pass the pre-built identityMap into recursive calls so the auto-build
|
|
213
323
|
// branch above is a no-op per bucket (would otherwise rebuild N times).
|
|
214
324
|
const inner = { ...config, timeBucket: undefined, timeBucketRange: undefined, identityMap };
|
|
215
325
|
const out = [];
|
|
216
|
-
|
|
326
|
+
|
|
327
|
+
// v1.6.0 (P1.6.5): step-0-anchored trend types. Mixpanel evaluates each
|
|
328
|
+
// trend interval as "step 0 in [start, stop); steps 1+ in
|
|
329
|
+
// [start, stop + conversion window)" (funnel_query.cpp:1398-1401), and
|
|
330
|
+
// retention's birth-in-interval with returns spilling past the interval
|
|
331
|
+
// edge (history.cpp:437-440). Plain event-partitioning would truncate
|
|
332
|
+
// any window spanning a bucket edge, so:
|
|
333
|
+
// - retention: run each bucket over the FULL stream, intersecting
|
|
334
|
+
// `cohortWindow` with the bucket bounds (births anchor in-bucket,
|
|
335
|
+
// returns spill freely). ARB keeps a separate retention history
|
|
336
|
+
// per (user, interval) — a user re-births in EVERY interval that
|
|
337
|
+
// contains a cohort-side match (retention_query.cpp:1334-1341), so
|
|
338
|
+
// the same user can appear in multiple buckets' cohorts;
|
|
339
|
+
// - funnelFrequency / timeToConvert: slice to [start, stop + window)
|
|
340
|
+
// and pass `anchorRange` down so step 0 only anchors in
|
|
341
|
+
// [start, stop).
|
|
342
|
+
// v1.6.0 (P1.6.6): every funnel order except 'middle-fixed' now routes
|
|
343
|
+
// through the engine (anyOrder blocks honor anchorRange — the funnel's
|
|
344
|
+
// FIRST POSITION, whatever slot fills it, must anchor in [start, stop)).
|
|
345
|
+
// 'middle-fixed' keeps plain partitioning: its scrambled slots are the
|
|
346
|
+
// two ends (non-contiguous), so it stays on the set-membership helper,
|
|
347
|
+
// which has no anchor concept.
|
|
348
|
+
const anchorableOrder = config.funnelOrder !== 'middle-fixed';
|
|
349
|
+
const anchored = config.type === 'retention'
|
|
350
|
+
|| ((config.type === 'funnelFrequency' || config.type === 'timeToConvert') && anchorableOrder);
|
|
351
|
+
|
|
352
|
+
if (anchored) {
|
|
353
|
+
for (const { period } of buckets) {
|
|
354
|
+
const { startMs, endMs } = bucketBoundsMs(period, config.timeBucket);
|
|
355
|
+
let rows;
|
|
356
|
+
if (config.type === 'retention') {
|
|
357
|
+
const cw = config.cohortWindow || {};
|
|
358
|
+
const from = Math.max(startMs, cw.from != null ? toMs(cw.from) : -Infinity);
|
|
359
|
+
const to = Math.min(endMs - 1, cw.to != null ? toMs(cw.to) : Infinity); // cohortWindow bounds are inclusive
|
|
360
|
+
rows = from > to ? [] : emulateBreakdown(events, { ...inner, cohortWindow: { from, to } });
|
|
361
|
+
} else {
|
|
362
|
+
const spillMs = typeof config.conversionWindowMs === 'number'
|
|
363
|
+
? config.conversionWindowMs
|
|
364
|
+
: (config.conversionWindow && typeof config.conversionWindow.n === 'number'
|
|
365
|
+
// session windows: ARB's per-step check is ordinal-only, but
|
|
366
|
+
// histories terminate conversion_window_max_length_seconds
|
|
367
|
+
// (n × SECONDS_PER_DAY for sessions, unit.c:14) past the
|
|
368
|
+
// INTERVAL END (funnel_query.cpp:1620) — this slice mirrors that.
|
|
369
|
+
? config.conversionWindow.n * 86400_000
|
|
370
|
+
: Infinity);
|
|
371
|
+
const sliceEnd = endMs + spillMs;
|
|
372
|
+
const slice = events.filter(e => {
|
|
373
|
+
const t = toMs(e && e.time);
|
|
374
|
+
return Number.isFinite(t) && t >= startMs && t < sliceEnd;
|
|
375
|
+
});
|
|
376
|
+
rows = emulateBreakdown(slice, { ...inner, anchorRange: { fromMs: startMs, toMs: endMs } });
|
|
377
|
+
}
|
|
378
|
+
if (rows.length) {
|
|
379
|
+
for (const r of rows) out.push({ period, ...r });
|
|
380
|
+
} else {
|
|
381
|
+
out.push({ period, _empty: true });
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
return out;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// v1.6.0 fix round (B5): firstTimeOnly must see the FULL stream.
|
|
388
|
+
// Mixpanel computes ONE first_event_time per user over the whole
|
|
389
|
+
// lookback (~5y — event_selector.py:125-149). Recursing with the raw
|
|
390
|
+
// per-bucket slice would re-elect a "first" inside every bucket,
|
|
391
|
+
// counting each active user once per bucket instead of exactly once
|
|
392
|
+
// overall. Filter the FULL stream, repartition, clear the flag for
|
|
393
|
+
// the inner calls — the survivor lands in its one true bucket.
|
|
394
|
+
//
|
|
395
|
+
// countType 'sessions' needs NO such hoist: sessions never cross UTC
|
|
396
|
+
// midnight (unconditional daySplit, sessionize.js buildUserSessions —
|
|
397
|
+
// ARB resets session state at the day boundary) and every timeBucket
|
|
398
|
+
// unit (day/week/month) cuts at midnights, so each session's events
|
|
399
|
+
// are wholly inside one bucket and slice re-derivation reproduces the
|
|
400
|
+
// full-stream boundaries exactly (locked by the B5 invariant tests in
|
|
401
|
+
// event-breakdown.test.js).
|
|
402
|
+
let genericBuckets = buckets;
|
|
403
|
+
if (config.type === 'eventBreakdown' && config.firstTimeOnly) {
|
|
404
|
+
const firsts = filterFirstTimeEver(events, { event: config.event, identityMap });
|
|
405
|
+
genericBuckets = partitionByTimeBucket(firsts, config.timeBucket, range);
|
|
406
|
+
inner.firstTimeOnly = false;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
for (const { period, events: evs } of genericBuckets) {
|
|
217
410
|
const rows = emulateBreakdown(evs, inner);
|
|
218
411
|
if (rows.length) {
|
|
219
412
|
for (const r of rows) out.push({ period, ...r });
|
|
@@ -234,6 +427,10 @@ export function emulateBreakdown(events, config) {
|
|
|
234
427
|
case 'sessionMetrics': return sessionMetrics(events, /** @type {*} */ (cfg));
|
|
235
428
|
case 'retention': return retention(events, /** @type {*} */ (cfg));
|
|
236
429
|
case 'distinctCount': return distinctCount(events, /** @type {*} */ (cfg));
|
|
430
|
+
case 'eventBreakdown': return eventBreakdown(events, /** @type {*} */ (cfg));
|
|
431
|
+
case 'uniques': return uniques(events, /** @type {*} */ (cfg));
|
|
432
|
+
case 'lifecycle': return lifecycle(events, /** @type {*} */ (cfg));
|
|
433
|
+
case 'topPaths': return topPaths(events, /** @type {*} */ (cfg));
|
|
237
434
|
default: throw new Error(`emulateBreakdown: unknown type "${config.type}"`);
|
|
238
435
|
}
|
|
239
436
|
}
|
|
@@ -267,25 +464,130 @@ function frequencyByFrequency(events, { metricEvent, breakdownByFrequencyOf, per
|
|
|
267
464
|
// Step progression uses the greedy single-pass funnel engine
|
|
268
465
|
// (funnel-engine.js → history.cpp). Cohort breakdown axis uses distinct-period
|
|
269
466
|
// counting (addiction_query.cpp).
|
|
467
|
+
//
|
|
468
|
+
// v1.6.0 (P1.6.4): rows carry an `excluded` column — histories terminated by
|
|
469
|
+
// an exclusion step tally at step_index = the boundary they failed to cross
|
|
470
|
+
// (ARB `fr->excluded[reached + 1]`, funnel_query.cpp:3422; uniques variant
|
|
471
|
+
// `fr->excluded_uniques[reached + 1]`, :3337). A boundary with exclusions but
|
|
472
|
+
// zero conversions still emits a row.
|
|
270
473
|
|
|
271
|
-
function funnelFrequency(events, { steps, breakdownByFrequencyOf, conversionWindowMs, periodUnit = 'day', funnelOrder = 'sequential', identityMap, reentry, exclusionSteps, trackStepProperties, sessionScoped }) {
|
|
474
|
+
function funnelFrequency(events, { steps, breakdownByFrequencyOf, conversionWindowMs, conversionWindow, periodUnit = 'day', funnelOrder = 'sequential', identityMap, reentry, exclusionSteps, trackStepProperties, sessionScoped, countMode, holdPropertyConstant, anchorRange }) {
|
|
272
475
|
if (!Array.isArray(steps) || !steps.length) throw new Error('funnelFrequency requires steps[]');
|
|
273
476
|
if (!breakdownByFrequencyOf) throw new Error('funnelFrequency requires breakdownByFrequencyOf');
|
|
477
|
+
|
|
478
|
+
// v1.6.0 (findings #15): countMode 'totals'/'sessions' and holdPropertyConstant
|
|
479
|
+
// route to the greedy single-pass primitives directly — attempt histories and
|
|
480
|
+
// HPC sub-funnels have no analog in the any-order completion modes.
|
|
481
|
+
const isSequentialOrder = funnelOrder === 'sequential' || funnelOrder === 'interrupt' || funnelOrder === 'interrupted';
|
|
482
|
+
const isTotalsMode = countMode === 'totals' || countMode === 'sessions';
|
|
483
|
+
if ((isTotalsMode || holdPropertyConstant) && !isSequentialOrder) {
|
|
484
|
+
throw new Error(`funnelFrequency: countMode '${countMode}' / holdPropertyConstant require a sequential funnel order (got '${funnelOrder}')`);
|
|
485
|
+
}
|
|
486
|
+
// HPC buckets events by property value BEFORE evaluation (evaluateFunnelHPC),
|
|
487
|
+
// but Mixpanel derives session boundaries from the user's FULL event stream —
|
|
488
|
+
// sessionizing each bucket independently would merge across the gaps left by
|
|
489
|
+
// removed events and produce wrong ordinals. Refuse rather than mis-count.
|
|
490
|
+
if (holdPropertyConstant && (countMode === 'sessions' || conversionWindow)) {
|
|
491
|
+
throw new Error('funnelFrequency: holdPropertyConstant cannot combine with session-count conversion windows — session boundaries derive from the full event stream, but HPC evaluates per-property-value event subsets');
|
|
492
|
+
}
|
|
493
|
+
|
|
274
494
|
const userEvents = groupByUser(events, identityMap);
|
|
275
495
|
const result = [];
|
|
276
496
|
const conversions = new Map(); // `${stepIdx}|${b}` → count
|
|
497
|
+
const excludedCounts = new Map(); // `${stepIdx}|${b}` → count (stepIdx = excludedAtStep)
|
|
498
|
+
const funnelOpts = { conversionWindowMs, conversionWindow, countMode, reentry, exclusionSteps, trackStepProperties, sessionScoped, anchorRange };
|
|
277
499
|
for (const [, evs] of userEvents) {
|
|
278
|
-
//
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
500
|
+
// Frequency axis: under a trend interval (anchorRange set by the
|
|
501
|
+
// timeBucket wrapper) count only in-interval events — spill events past
|
|
502
|
+
// the bucket edge exist solely for later-step matching, not the cohort
|
|
503
|
+
// axis (which would otherwise shift users between frequency cohorts
|
|
504
|
+
// depending on the conversion window length).
|
|
505
|
+
const axisEvs = anchorRange
|
|
506
|
+
? evs.filter(e => {
|
|
507
|
+
const t = toMs(e && e.time);
|
|
508
|
+
return Number.isFinite(t)
|
|
509
|
+
&& (anchorRange.fromMs == null || t >= anchorRange.fromMs)
|
|
510
|
+
&& (anchorRange.toMs == null || t < anchorRange.toMs);
|
|
511
|
+
})
|
|
512
|
+
: evs;
|
|
513
|
+
const b = countDistinctPeriods(axisEvs, breakdownByFrequencyOf, /** @type {*} */ (periodUnit));
|
|
514
|
+
// One entry per credited history: uniques = the user's single (or best-
|
|
515
|
+
// across-HPC-values) progression; totals/sessions = every attempt.
|
|
516
|
+
/** @type {number[]} */
|
|
517
|
+
const reachedList = [];
|
|
518
|
+
// v1.6.0 (P1.6.4): exclusion boundary indexes (= reached + 1 of a
|
|
519
|
+
// terminated history) — ARB counts them per step slot:
|
|
520
|
+
// `fr->excluded[reached + 1]` (funnel_query.cpp:3422, totals) /
|
|
521
|
+
// `fr->excluded_uniques[reached + 1]` (funnel_query.cpp:3337, uniques).
|
|
522
|
+
/** @type {number[]} */
|
|
523
|
+
const excludedList = [];
|
|
524
|
+
if (holdPropertyConstant) {
|
|
525
|
+
const perValue = evaluateFunnelHPC(evs, steps, holdPropertyConstant, funnelOpts);
|
|
526
|
+
if (isTotalsMode) {
|
|
527
|
+
// Every attempt in every value bucket counts (totals semantics).
|
|
528
|
+
for (const r of perValue.values()) {
|
|
529
|
+
for (const a of /** @type {Array<*>} */ (r)) {
|
|
530
|
+
reachedList.push(a.reached);
|
|
531
|
+
if (a.terminatedByExclusion) excludedList.push(a.excludedAtStep);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
} else {
|
|
535
|
+
// Uniques: per-step distinct-user counting over parallel histories.
|
|
536
|
+
// A user appears at step s when ANY value thread reached ≥ s, and
|
|
537
|
+
// progression is contiguous from 0 — so the union of credited steps
|
|
538
|
+
// is 0..max(reached). Credit the user ONCE at the furthest thread.
|
|
539
|
+
let best = -1;
|
|
540
|
+
let bestExcluded = false;
|
|
541
|
+
for (const r of perValue.values()) {
|
|
542
|
+
const reached = /** @type {*} */ (r).reached;
|
|
543
|
+
if (reached > best) {
|
|
544
|
+
best = reached;
|
|
545
|
+
bestExcluded = !!(/** @type {*} */ (r).terminatedByExclusion);
|
|
546
|
+
} else if (reached === best && /** @type {*} */ (r).terminatedByExclusion) {
|
|
547
|
+
// ARB-UNCERTAIN: HPC exclusion uniques. Conservative reading —
|
|
548
|
+
// the user counts as excluded when ANY furthest-reaching value
|
|
549
|
+
// thread was terminated by exclusion (each per-value history
|
|
550
|
+
// aggregates independently in ARB; per-user dedup across
|
|
551
|
+
// threads at the same boundary is our distinct-user layer).
|
|
552
|
+
bestExcluded = true;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
if (best >= 0) {
|
|
556
|
+
reachedList.push(best);
|
|
557
|
+
if (bestExcluded) excludedList.push(best + 1);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
} else if (isTotalsMode) {
|
|
561
|
+
const attempts = /** @type {Array<*>} */ (evaluateFunnel(evs, steps, funnelOpts));
|
|
562
|
+
for (const a of attempts) {
|
|
563
|
+
reachedList.push(a.reached);
|
|
564
|
+
if (a.terminatedByExclusion) excludedList.push(a.excludedAtStep);
|
|
565
|
+
}
|
|
566
|
+
} else {
|
|
567
|
+
// v1.5: dispatch on funnel.order so non-sequential modes don't return 0% trivially.
|
|
568
|
+
const r = evaluateFunnelByOrder(evs, steps, { conversionWindowMs, conversionWindow, funnelOrder, reentry, exclusionSteps, trackStepProperties, sessionScoped, anchorRange });
|
|
569
|
+
reachedList.push(r.reached);
|
|
570
|
+
if (/** @type {*} */ (r).terminatedByExclusion) excludedList.push(/** @type {*} */ (r).excludedAtStep);
|
|
571
|
+
}
|
|
572
|
+
for (const reached of reachedList) {
|
|
573
|
+
for (let s = 0; s <= reached; s++) {
|
|
574
|
+
const key = `${s}|${b}`;
|
|
575
|
+
conversions.set(key, (conversions.get(key) || 0) + 1);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
for (const s of excludedList) {
|
|
282
579
|
const key = `${s}|${b}`;
|
|
283
|
-
|
|
580
|
+
excludedCounts.set(key, (excludedCounts.get(key) || 0) + 1);
|
|
284
581
|
}
|
|
285
582
|
}
|
|
286
|
-
|
|
583
|
+
const rowKeys = new Set([...conversions.keys(), ...excludedCounts.keys()]);
|
|
584
|
+
for (const key of rowKeys) {
|
|
287
585
|
const [s, b] = key.split('|').map(Number);
|
|
288
|
-
result.push({
|
|
586
|
+
result.push({
|
|
587
|
+
step: steps[s], step_index: s, breakdown_freq: b,
|
|
588
|
+
conversions: conversions.get(key) || 0, conversion_pct: 0,
|
|
589
|
+
excluded: excludedCounts.get(key) || 0,
|
|
590
|
+
});
|
|
289
591
|
}
|
|
290
592
|
// Conversion % at each step relative to its own breakdown_freq's step-0 baseline.
|
|
291
593
|
const baseline = new Map();
|
|
@@ -359,14 +661,38 @@ function aggregatePerUser(events, { event, property, agg = 'avg', breakdownByFre
|
|
|
359
661
|
|
|
360
662
|
// ── Time to Convert (Funnel TTC, broken down by user property) ──
|
|
361
663
|
//
|
|
362
|
-
//
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
//
|
|
366
|
-
//
|
|
664
|
+
// Steps matched via the greedy funnel engine (history.cpp). v1.6.0 (P1.6.7)
|
|
665
|
+
// aligns the arithmetic with ARB's funnel aggregation
|
|
666
|
+
// (funnel_query.cpp:3355-3380) and extends past 2-step pairs: pass `steps`
|
|
667
|
+
// (>= 2 entries, funnelOrder/exclusions apply) or keep the
|
|
668
|
+
// `fromEvent`/`toEvent` pair sugar.
|
|
669
|
+
//
|
|
670
|
+
// Output per segment row (rows are emitted for segments with >= 1 full
|
|
671
|
+
// conversion — the TTC chart is a converted-users report):
|
|
672
|
+
// - avg_ttc_ms / median_ttc_ms — over converted users; ttcMs is `$ttc` at
|
|
673
|
+
// ms resolution (MAX over slot times − position-0 time,
|
|
674
|
+
// history.cpp:914-918).
|
|
675
|
+
// - avg_ttc_s — mean of the integer-second `$ttc` values (floor'd per user
|
|
676
|
+
// BEFORE averaging, matching the computed property's resolution).
|
|
677
|
+
// - step_counts[i] — attempts that reached position i, drop-offs included
|
|
678
|
+
// (fr->counts, funnel_query.cpp:3362 — the loop runs for every history).
|
|
679
|
+
// - gap_avg_s[g] / cumulative_avg_s[g] — sum_deltas[i]/counts[i] and
|
|
680
|
+
// sum_deltas_from_start[i]/counts[i] for position i = g+1
|
|
681
|
+
// (funnel_query.cpp:3374-3380): per-gap deltas are integer seconds
|
|
682
|
+
// clamped to 0 per gap; the cumulative track sums the CLAMPED deltas.
|
|
683
|
+
// `null` when no attempt reached that position.
|
|
367
684
|
|
|
368
|
-
function timeToConvert(events, { fromEvent, toEvent, breakdownByUserProperty, profiles = [], funnelOrder = 'sequential', conversionWindowMs, identityMap, reentry, exclusionSteps, sessionScoped }) {
|
|
369
|
-
|
|
685
|
+
function timeToConvert(events, { fromEvent, toEvent, steps: stepsOpt, breakdownByUserProperty, profiles = [], funnelOrder = 'sequential', conversionWindowMs, identityMap, reentry, exclusionSteps, sessionScoped, anchorRange }) {
|
|
686
|
+
let steps;
|
|
687
|
+
if (Array.isArray(stepsOpt) && stepsOpt.length) {
|
|
688
|
+
if (fromEvent || toEvent) throw new Error('timeToConvert takes either steps[] or fromEvent/toEvent, not both');
|
|
689
|
+
if (stepsOpt.length < 2) throw new Error('timeToConvert steps[] needs at least 2 steps');
|
|
690
|
+
steps = stepsOpt;
|
|
691
|
+
} else {
|
|
692
|
+
if (!fromEvent || !toEvent) throw new Error('timeToConvert requires fromEvent and toEvent (or steps[])');
|
|
693
|
+
steps = [fromEvent, toEvent];
|
|
694
|
+
}
|
|
695
|
+
const numSteps = steps.length;
|
|
370
696
|
const userEvents = groupByUser(events, identityMap);
|
|
371
697
|
const profileByUid = new Map();
|
|
372
698
|
for (const p of profiles) {
|
|
@@ -374,62 +700,117 @@ function timeToConvert(events, { fromEvent, toEvent, breakdownByUserProperty, pr
|
|
|
374
700
|
const uid = p.distinct_id || p.user_id;
|
|
375
701
|
if (uid) profileByUid.set(uid, p);
|
|
376
702
|
}
|
|
377
|
-
|
|
703
|
+
// segValue → { ttcs, ttcSecs, counts, sumGap, sumCum }
|
|
704
|
+
const buckets = new Map();
|
|
705
|
+
const acc = (segValue) => {
|
|
706
|
+
if (!buckets.has(segValue)) buckets.set(segValue, {
|
|
707
|
+
ttcs: [], ttcSecs: [],
|
|
708
|
+
counts: new Array(numSteps).fill(0),
|
|
709
|
+
sumGap: new Array(numSteps - 1).fill(0),
|
|
710
|
+
sumCum: new Array(numSteps - 1).fill(0),
|
|
711
|
+
});
|
|
712
|
+
return buckets.get(segValue);
|
|
713
|
+
};
|
|
378
714
|
for (const [uid, evs] of userEvents) {
|
|
379
|
-
// v1.5: respect funnel.order.
|
|
380
|
-
// (
|
|
381
|
-
const r = evaluateFunnelByOrder(evs,
|
|
382
|
-
if (!r.completed || r.ttcMs === null || !Number.isFinite(r.ttcMs) || r.ttcMs < 0) continue;
|
|
715
|
+
// v1.5: respect funnel.order. middle-fixed stays set-membership
|
|
716
|
+
// (verificationKind 'partial') — no gap arrays there.
|
|
717
|
+
const r = evaluateFunnelByOrder(evs, steps, { funnelOrder, conversionWindowMs, reentry, exclusionSteps, sessionScoped, anchorRange });
|
|
383
718
|
const profile = profileByUid.get(uid);
|
|
384
719
|
const segValue = breakdownByUserProperty
|
|
385
720
|
? (profile ? (profile[breakdownByUserProperty] ?? 'unknown') : 'unknown')
|
|
386
721
|
: 'all';
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
722
|
+
const a = acc(segValue);
|
|
723
|
+
// funnel_query.cpp:3359-3380 — EVERY attempt (converted or not)
|
|
724
|
+
// records counts[i] for i <= reached and gap sums for 1 <= i <= reached.
|
|
725
|
+
if (typeof r.reached === 'number' && r.reached >= 0 && Array.isArray(r.gapSeconds)) {
|
|
726
|
+
for (let p = 0; p <= r.reached && p < numSteps; p++) a.counts[p]++;
|
|
727
|
+
for (let g = 0; g < r.gapSeconds.length && g < numSteps - 1; g++) {
|
|
728
|
+
a.sumGap[g] += r.gapSeconds[g];
|
|
729
|
+
a.sumCum[g] += r.gapSecondsFromStart[g];
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
if (!r.completed || r.ttcMs === null || !Number.isFinite(r.ttcMs) || r.ttcMs < 0) continue;
|
|
733
|
+
a.ttcs.push(r.ttcMs);
|
|
734
|
+
// $ttc is integer seconds (history.cpp:917-918); middle-fixed partial
|
|
735
|
+
// results carry no ttcSeconds — floor their informational ttcMs.
|
|
736
|
+
a.ttcSecs.push(typeof r.ttcSeconds === 'number' ? r.ttcSeconds : Math.floor(r.ttcMs / 1000));
|
|
737
|
+
}
|
|
738
|
+
return [...buckets.entries()]
|
|
739
|
+
.filter(([, a]) => a.ttcs.length > 0)
|
|
740
|
+
.map(([seg, a]) => ({
|
|
741
|
+
segment_value: seg,
|
|
742
|
+
user_count: a.ttcs.length,
|
|
743
|
+
avg_ttc_ms: a.ttcs.reduce((x, y) => x + y, 0) / a.ttcs.length,
|
|
744
|
+
median_ttc_ms: median(a.ttcs),
|
|
745
|
+
avg_ttc_s: a.ttcSecs.reduce((x, y) => x + y, 0) / a.ttcSecs.length,
|
|
746
|
+
step_counts: a.counts,
|
|
747
|
+
gap_avg_s: a.sumGap.map((s, g) => a.counts[g + 1] > 0 ? s / a.counts[g + 1] : null),
|
|
748
|
+
cumulative_avg_s: a.sumCum.map((s, g) => a.counts[g + 1] > 0 ? s / a.counts[g + 1] : null),
|
|
749
|
+
}))
|
|
750
|
+
.sort((x, y) => String(x.segment_value).localeCompare(String(y.segment_value)));
|
|
396
751
|
}
|
|
397
752
|
|
|
398
753
|
// ── Attributed By (first-/last-touch attribution by event property value) ──
|
|
399
754
|
//
|
|
400
|
-
//
|
|
401
|
-
// (
|
|
402
|
-
//
|
|
403
|
-
//
|
|
755
|
+
// NO touchpoint cap for FIRST/LAST models. `TOUCHPOINTS_LIMIT = 10`
|
|
756
|
+
// (backend/libquery/properties_over_time/attributed_value_reader.cpp:16)
|
|
757
|
+
// rides in whoval_reader_params on every read, but the FIRST/LAST paths
|
|
758
|
+
// execute hard-`LIMIT 1` statements (whoval/read.cpp:173-192 first_stmt_,
|
|
759
|
+
// :643-655 get_first/last_value) — only `sorted_list_stmt_` (`LIMIT ?4`,
|
|
760
|
+
// read.cpp:595, get_list_values :681-685) consumes the cap, and that
|
|
761
|
+
// statement serves multi-touch list models (LINEAR/PARTICIPATION/
|
|
762
|
+
// TIME_DECAY) this emulator doesn't implement. FIRST = globally first
|
|
763
|
+
// touch in the lookback window, however many touches precede it.
|
|
764
|
+
//
|
|
765
|
+
// Per-conversion semantics: Mixpanel runs attribution once PER conversion
|
|
766
|
+
// event — `attributed_value_reader_read` takes a single `event_time_ms` and
|
|
767
|
+
// builds a fresh lookback read ending at that conversion.
|
|
768
|
+
// `perConversion: 'all'` matches that; the default `'first'` keeps the v1.5
|
|
769
|
+
// one-conversion-per-user behavior for back-compat.
|
|
770
|
+
//
|
|
771
|
+
// ⚠ TOUCHPOINT SEAM (findings #2, closed-as-documented): the GENERATOR
|
|
772
|
+
// samples which events get UTM stamps uniformly across a user's lifetime
|
|
773
|
+
// (maxTouchpointsPerUser sampling in lib/orchestrators/user-loop.js —
|
|
774
|
+
// `sampled across lifetime`), while this verifier and Mixpanel both read
|
|
775
|
+
// touchpoints BEFORE each conversion. A user with more eligible events
|
|
776
|
+
// than the generator cap can have touches that never got stamped — and
|
|
777
|
+
// stamped touches can postdate every conversion. Divergence is
|
|
778
|
+
// theoretical below ~10 eligible events per user (the generator cap);
|
|
779
|
+
// attribution-engineering hooks should OVERWRITE engine-stamped UTMs near
|
|
780
|
+
// the conversion rather than relying on the lifetime-uniform sampling
|
|
781
|
+
// (hook rule 10, CLAUDE.md).
|
|
404
782
|
|
|
405
783
|
function attributedBy(events, {
|
|
406
784
|
conversionEvent,
|
|
407
785
|
attributionEvent,
|
|
408
786
|
attributionProperty,
|
|
409
787
|
model = 'firstTouch',
|
|
410
|
-
|
|
788
|
+
perConversion = 'first',
|
|
411
789
|
identityMap,
|
|
412
790
|
}) {
|
|
413
791
|
if (!conversionEvent || !attributionEvent || !attributionProperty) {
|
|
414
792
|
throw new Error('attributedBy requires conversionEvent, attributionEvent, attributionProperty');
|
|
415
793
|
}
|
|
794
|
+
if (perConversion !== 'first' && perConversion !== 'all') {
|
|
795
|
+
throw new Error(`attributedBy: unknown perConversion "${perConversion}" — use 'first' or 'all'`);
|
|
796
|
+
}
|
|
416
797
|
const userEvents = groupByUser(events, identityMap);
|
|
417
798
|
const counts = new Map();
|
|
418
799
|
for (const [, evs] of userEvents) {
|
|
419
800
|
const sorted = sortByTime(evs);
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
const
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
801
|
+
const conversions = perConversion === 'all'
|
|
802
|
+
? sorted.filter(e => e && e.event === conversionEvent)
|
|
803
|
+
: sorted.filter(e => e && e.event === conversionEvent).slice(0, 1);
|
|
804
|
+
for (const conversion of conversions) {
|
|
805
|
+
const conversionTime = toMs(conversion.time);
|
|
806
|
+
const allTouches = sorted.filter(e =>
|
|
807
|
+
e && e.event === attributionEvent && toMs(e.time) <= conversionTime
|
|
808
|
+
);
|
|
809
|
+
if (!allTouches.length) continue;
|
|
810
|
+
const touch = model === 'lastTouch' ? allTouches[allTouches.length - 1] : allTouches[0];
|
|
811
|
+
const v = touch[attributionProperty] ?? 'unknown';
|
|
812
|
+
counts.set(v, (counts.get(v) || 0) + 1);
|
|
813
|
+
}
|
|
433
814
|
}
|
|
434
815
|
return [...counts.entries()].map(([source, count]) => ({
|
|
435
816
|
attribution_value: source,
|
|
@@ -437,85 +818,298 @@ function attributedBy(events, {
|
|
|
437
818
|
})).sort((a, b) => b.conversions - a.conversions);
|
|
438
819
|
}
|
|
439
820
|
|
|
440
|
-
// ── Retention (birth-anchored
|
|
821
|
+
// ── Retention (birth-anchored buckets) ───────────────────────────────────────
|
|
441
822
|
//
|
|
442
823
|
// Reference: backend/arb/reader/queries/retention_query.cpp
|
|
443
824
|
//
|
|
444
|
-
// Bucketing rule (retention_query.cpp:
|
|
445
|
-
// time_to_retention_event_s = retention_event_time_s -
|
|
825
|
+
// Bucketing rule (retention_query.cpp:1258-1262):
|
|
826
|
+
// time_to_retention_event_s = retention_event_time_s - aligned_birth_time_s
|
|
446
827
|
// bucket = floor(time_to_retention_event_s / bucket_seconds)
|
|
828
|
+
// Bucket seconds per unit (libquery/time/unit.c:5-16 + libquery/util.h:265-273):
|
|
829
|
+
// hour = 3600s, day = 86400s, week = 7d, month = 31d FIXED ("maximum
|
|
830
|
+
// possible seconds in a month" — NOT calendar months).
|
|
831
|
+
//
|
|
832
|
+
// Birth-vs-return gate (retention_query.cpp:1120-1139,
|
|
833
|
+
// retention_query_event_occurs_after_birth — cites COR-233):
|
|
834
|
+
// `<=` applies ONLY when birthCanRetain AND the return event ALSO matches
|
|
835
|
+
// the birth filter (`matches_first`). Distinct birth/return events at the
|
|
836
|
+
// same ms stay strictly `<` even with birthCanRetain: true. The gate reads
|
|
837
|
+
// the RAW birth time — calendar alignment applies only inside the bucket
|
|
838
|
+
// delta (retention_query_get_aligned_event_time_s at :1260).
|
|
839
|
+
//
|
|
840
|
+
// Compounded (retention_query.cpp:677-685): `rq->second = rq->first` — the
|
|
841
|
+
// return side IS the cohort side, so every cohort event is a return
|
|
842
|
+
// candidate ("DAU coming back").
|
|
843
|
+
//
|
|
844
|
+
// Unbounded modes:
|
|
845
|
+
// carryForward — read-time carry (:1854-1868): retained at N if active in
|
|
846
|
+
// ANY bucket ≤ N.
|
|
847
|
+
// carryBack — reverse-iteration carry (:274-278): retained at N if active
|
|
848
|
+
// in ANY bucket ≥ N.
|
|
849
|
+
// consecutiveForward — gated at WRITE time (:1275-1287): seen_in[N] is
|
|
850
|
+
// written only if seen_in[N−1] is already set (except N = 0). Since a
|
|
851
|
+
// user's returns arrive time-ordered, the surviving marks are exactly
|
|
852
|
+
// the maximal consecutive prefix {0..k} of buckets hit.
|
|
447
853
|
//
|
|
448
|
-
//
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
//
|
|
454
|
-
//
|
|
455
|
-
//
|
|
456
|
-
//
|
|
457
|
-
//
|
|
458
|
-
//
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
//
|
|
465
|
-
//
|
|
466
|
-
//
|
|
467
|
-
//
|
|
468
|
-
//
|
|
469
|
-
//
|
|
470
|
-
//
|
|
471
|
-
//
|
|
472
|
-
//
|
|
473
|
-
//
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
854
|
+
// calendarStart (retention_query.cpp:312-332, applied to the birth at :1260):
|
|
855
|
+
// ARB steps in fixed unit_s increments from the query interval start;
|
|
856
|
+
// production intervals start on unit boundaries, so the effect is flooring
|
|
857
|
+
// the BIRTH time to the bucket-unit boundary (week = ISO Monday, matching
|
|
858
|
+
// partitionByTimeBucket) before computing deltas. Month alignment floors to
|
|
859
|
+
// the calendar month start while the bucket WIDTH stays 31d fixed.
|
|
860
|
+
//
|
|
861
|
+
// Internal-event ignore list (retention_query.cpp:2546-2555): when a side has
|
|
862
|
+
// no explicit event selector (null / '$any_event'), $campaign_delivery,
|
|
863
|
+
// $campaign_bounced, $create_alias, $identify, $merge are ignored for that
|
|
864
|
+
// side. Explicit selectors bypass the list.
|
|
865
|
+
//
|
|
866
|
+
// segmentOn 'return' (SEGMENT_EVENT_SECOND, retention_query.cpp:1421-1444):
|
|
867
|
+
// the segment value is read from each RETURN event; a user joins a segment's
|
|
868
|
+
// cohort only via a qualifying return in that segment — births are
|
|
869
|
+
// unsegmented unless birthCanRetain lets the birth itself qualify as a
|
|
870
|
+
// return (:1413-1419). ARB reads profile/SCD-style segment properties as-of
|
|
871
|
+
// the BIRTH time even in this mode (:1421-1444); this emulator segments on
|
|
872
|
+
// flat event properties only, so that read-path does not arise here.
|
|
873
|
+
//
|
|
874
|
+
// Unrecognized option keys THROW (1.6.0 behavior change — kills the
|
|
875
|
+
// silent-ignore class of bug where e.g. `compounded: true` was dropped).
|
|
876
|
+
//
|
|
877
|
+
// timeBucket composition caveat: the generic wrapper partitions EVENTS, so
|
|
878
|
+
// returns crossing a bucket edge are truncated. P1.6.5 re-anchors retention
|
|
879
|
+
// buckets on the BIRTH event time.
|
|
880
|
+
|
|
881
|
+
const RETENTION_UNIT_MS = {
|
|
882
|
+
hour: 3600 * 1000,
|
|
883
|
+
day: 86400 * 1000,
|
|
884
|
+
week: 7 * 86400 * 1000,
|
|
885
|
+
// "Maximum possible seconds in a month" — libquery/util.h:265-273.
|
|
886
|
+
month: 31 * 86400 * 1000,
|
|
887
|
+
};
|
|
888
|
+
|
|
889
|
+
const RETENTION_IGNORED_INTERNAL = new Set([
|
|
890
|
+
'$campaign_delivery', '$campaign_bounced', '$create_alias', '$identify', '$merge',
|
|
891
|
+
]);
|
|
892
|
+
|
|
893
|
+
const RETENTION_KNOWN_KEYS = new Set([
|
|
894
|
+
'type', 'profiles', 'identityMap', 'timeBucket', 'timeBucketRange',
|
|
895
|
+
'cohortEvent', 'returnEvent', 'cohortWhere', 'returnWhere', 'compounded',
|
|
896
|
+
'dayBuckets', 'bucketUnit', 'unbounded', 'carry_forward', 'bucketAlignment',
|
|
897
|
+
'cohortWindow', 'segmentBy', 'segmentOn', 'birthCanRetain',
|
|
898
|
+
]);
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* Does event `e` match one side (cohort or return) of the retention query?
|
|
902
|
+
* `sideEvent` null / '$any_event' = any event EXCEPT the internal ignore
|
|
903
|
+
* list (retention_query.cpp:2546-2555 — only applies when the side has no
|
|
904
|
+
* explicit selectors).
|
|
905
|
+
*/
|
|
906
|
+
function retentionSideMatches(e, sideEvent, sideWhere) {
|
|
907
|
+
if (sideEvent == null || sideEvent === '$any_event') {
|
|
908
|
+
if (RETENTION_IGNORED_INTERNAL.has(e.event)) return false;
|
|
909
|
+
} else if (e.event !== sideEvent) {
|
|
910
|
+
return false;
|
|
911
|
+
}
|
|
912
|
+
return matchesWhere(e, sideWhere);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* Floor `ms` to the start of its bucket unit (UTC). Week = ISO Monday,
|
|
917
|
+
* matching partitionByTimeBucket; month = calendar month start.
|
|
918
|
+
*/
|
|
919
|
+
function retentionFloorToUnit(ms, unit) {
|
|
920
|
+
const DAY = RETENTION_UNIT_MS.day;
|
|
921
|
+
switch (unit) {
|
|
922
|
+
case 'hour': return ms - (ms % RETENTION_UNIT_MS.hour);
|
|
923
|
+
case 'day': return ms - (ms % DAY);
|
|
924
|
+
case 'week': {
|
|
925
|
+
const dayStart = ms - (ms % DAY);
|
|
926
|
+
const dow = new Date(dayStart).getUTCDay(); // 0 = Sunday
|
|
927
|
+
return dayStart - ((dow + 6) % 7) * DAY; // back to ISO Monday
|
|
928
|
+
}
|
|
929
|
+
case 'month': {
|
|
930
|
+
const d = new Date(ms);
|
|
931
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1);
|
|
932
|
+
}
|
|
933
|
+
default: throw new Error(`retention: unknown bucketUnit "${unit}"`);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
function retention(events, cfg) {
|
|
938
|
+
// Strict option keys — unknown keys were previously silently ignored
|
|
939
|
+
// (declared 1.6.0 behavior change).
|
|
940
|
+
for (const k of Object.keys(cfg)) {
|
|
941
|
+
if (!RETENTION_KNOWN_KEYS.has(k)) {
|
|
942
|
+
throw new Error(`retention: unrecognized option "${k}" (known: ${[...RETENTION_KNOWN_KEYS].join(', ')})`);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
const {
|
|
947
|
+
cohortEvent = null,
|
|
948
|
+
cohortWhere,
|
|
949
|
+
compounded = false,
|
|
950
|
+
dayBuckets = [1, 7, 14, 30],
|
|
951
|
+
bucketUnit = 'day',
|
|
952
|
+
bucketAlignment = 'birth',
|
|
953
|
+
cohortWindow,
|
|
954
|
+
segmentBy,
|
|
955
|
+
segmentOn = 'birth',
|
|
956
|
+
birthCanRetain = false,
|
|
957
|
+
identityMap,
|
|
958
|
+
} = cfg;
|
|
959
|
+
|
|
960
|
+
if (!('cohortEvent' in cfg)) {
|
|
961
|
+
throw new Error('retention requires cohortEvent (null or "$any_event" = any event)');
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
// Compounded: return side := cohort side (retention_query.cpp:677-685).
|
|
965
|
+
let returnEvent, returnWhere;
|
|
966
|
+
if (compounded) {
|
|
967
|
+
if ('returnEvent' in cfg && cfg.returnEvent !== cohortEvent) {
|
|
968
|
+
throw new Error('retention: compounded sets returnEvent := cohortEvent — remove the conflicting returnEvent');
|
|
969
|
+
}
|
|
970
|
+
// Same guard for returnWhere (fix-round nit 4): `rq->second = rq->first`
|
|
971
|
+
// replaces the WHOLE return side, filters included — a conflicting
|
|
972
|
+
// returnWhere would be silently discarded, the exact silent-ignore bug
|
|
973
|
+
// class the strict-keys throw exists to kill.
|
|
974
|
+
if ('returnWhere' in cfg && cfg.returnWhere !== cohortWhere) {
|
|
975
|
+
throw new Error('retention: compounded sets returnWhere := cohortWhere — remove the conflicting returnWhere');
|
|
976
|
+
}
|
|
977
|
+
returnEvent = cohortEvent;
|
|
978
|
+
returnWhere = cohortWhere;
|
|
979
|
+
} else {
|
|
980
|
+
if (!('returnEvent' in cfg)) {
|
|
981
|
+
throw new Error('retention requires returnEvent (or compounded: true)');
|
|
982
|
+
}
|
|
983
|
+
returnEvent = cfg.returnEvent;
|
|
984
|
+
returnWhere = cfg.returnWhere;
|
|
985
|
+
}
|
|
986
|
+
|
|
481
987
|
if (!Array.isArray(dayBuckets) || !dayBuckets.length) {
|
|
482
988
|
throw new Error('retention requires non-empty dayBuckets');
|
|
483
989
|
}
|
|
990
|
+
// Write horizon: the highest queried bucket index (see the bounds check
|
|
991
|
+
// at the mark loop — retention_query.cpp:1264).
|
|
992
|
+
const maxQueriedBucket = Math.max(...dayBuckets);
|
|
484
993
|
|
|
994
|
+
const unitMs = RETENTION_UNIT_MS[bucketUnit];
|
|
995
|
+
if (!unitMs) {
|
|
996
|
+
throw new Error(`retention: unknown bucketUnit "${bucketUnit}" (hour|day|week|month)`);
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
// `carry_forward` boolean is the deprecated alias for unbounded: 'carryForward'.
|
|
1000
|
+
let unbounded = cfg.unbounded;
|
|
1001
|
+
if (unbounded === undefined) unbounded = cfg.carry_forward ? 'carryForward' : 'none';
|
|
1002
|
+
if (!['none', 'carryForward', 'carryBack', 'consecutiveForward'].includes(unbounded)) {
|
|
1003
|
+
throw new Error(`retention: unknown unbounded mode "${unbounded}" (none|carryForward|carryBack|consecutiveForward)`);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
if (bucketAlignment !== 'birth' && bucketAlignment !== 'calendarStart') {
|
|
1007
|
+
throw new Error(`retention: unknown bucketAlignment "${bucketAlignment}" (birth|calendarStart)`);
|
|
1008
|
+
}
|
|
1009
|
+
if (segmentOn !== 'birth' && segmentOn !== 'return') {
|
|
1010
|
+
throw new Error(`retention: unknown segmentOn "${segmentOn}" (birth|return)`);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
let windowFromMs = -Infinity, windowToMs = Infinity;
|
|
1014
|
+
if (cohortWindow != null) {
|
|
1015
|
+
windowFromMs = toMs(cohortWindow.from);
|
|
1016
|
+
windowToMs = toMs(cohortWindow.to);
|
|
1017
|
+
if (!Number.isFinite(windowFromMs) || !Number.isFinite(windowToMs)) {
|
|
1018
|
+
throw new Error('retention: cohortWindow requires { from, to } timestamps');
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
const segmentedOnReturn = segmentOn === 'return' && !!segmentBy;
|
|
485
1023
|
const userEvents = groupByUser(events, identityMap);
|
|
486
1024
|
|
|
487
|
-
// segment → cohort users + per-user
|
|
1025
|
+
// segment → cohort users + per-user retained-bucket marks
|
|
488
1026
|
const cohorts = new Map();
|
|
489
1027
|
const ensureSegment = (seg) => {
|
|
490
|
-
if (!cohorts.has(seg)) cohorts.set(seg, { users: new Set(),
|
|
1028
|
+
if (!cohorts.has(seg)) cohorts.set(seg, { users: new Set(), markedByUser: new Map() });
|
|
491
1029
|
return cohorts.get(seg);
|
|
492
1030
|
};
|
|
493
1031
|
|
|
494
1032
|
for (const [uid, evs] of userEvents) {
|
|
495
|
-
// Birth = earliest
|
|
1033
|
+
// Birth = earliest cohort-side match INSIDE the cohort window
|
|
1034
|
+
// (inclusive bounds). ARB's in_birth_window gate
|
|
1035
|
+
// (retention_query.cpp:1188) means an out-of-window cohort event never
|
|
1036
|
+
// sets first_time (:1206-1208) — it does NOT consume the birth. A user
|
|
1037
|
+
// whose first-ever cohort event precedes the window still births at
|
|
1038
|
+
// their first in-window one. Under timeBucket the dispatcher runs each
|
|
1039
|
+
// bucket over the full stream with cohortWindow intersected to the
|
|
1040
|
+
// bucket bounds, so this in-window election is what realizes ARB's
|
|
1041
|
+
// per-(user, interval) retention history (:1334-1341): a user with
|
|
1042
|
+
// cohort events in two intervals re-births in each and appears in BOTH
|
|
1043
|
+
// cohorts, with returns bucketed relative to that interval's own birth.
|
|
496
1044
|
const sorted = sortByTime(evs);
|
|
497
|
-
|
|
1045
|
+
let birth = null, birthMs = NaN;
|
|
1046
|
+
for (const e of sorted) {
|
|
1047
|
+
if (!retentionSideMatches(e, cohortEvent, cohortWhere)) continue;
|
|
1048
|
+
const t = toMs(e.time);
|
|
1049
|
+
if (!Number.isFinite(t)) continue;
|
|
1050
|
+
if (t < windowFromMs || t > windowToMs) continue;
|
|
1051
|
+
birth = e;
|
|
1052
|
+
birthMs = t;
|
|
1053
|
+
break;
|
|
1054
|
+
}
|
|
498
1055
|
if (!birth) continue;
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
1056
|
+
|
|
1057
|
+
const alignedBirthMs = bucketAlignment === 'calendarStart'
|
|
1058
|
+
? retentionFloorToUnit(birthMs, bucketUnit)
|
|
1059
|
+
: birthMs;
|
|
1060
|
+
|
|
1061
|
+
// Collect this user's retained-bucket marks. Under segmentOn 'return'
|
|
1062
|
+
// each segment gets its own mark set (returns partition by their own
|
|
1063
|
+
// property value); otherwise one set under the birth-derived segment.
|
|
1064
|
+
const markedBySeg = new Map();
|
|
1065
|
+
const markSetFor = (seg) => {
|
|
1066
|
+
if (!markedBySeg.has(seg)) markedBySeg.set(seg, new Set());
|
|
1067
|
+
return markedBySeg.get(seg);
|
|
1068
|
+
};
|
|
1069
|
+
const birthSeg = segmentedOnReturn ? null
|
|
1070
|
+
: (segmentBy ? (birth[segmentBy] ?? 'unknown') : 'all');
|
|
1071
|
+
|
|
1072
|
+
for (const e of sorted) {
|
|
1073
|
+
if (!retentionSideMatches(e, returnEvent, returnWhere)) continue;
|
|
1074
|
+
const evMs = toMs(e.time);
|
|
510
1075
|
if (!Number.isFinite(evMs)) continue;
|
|
511
|
-
//
|
|
512
|
-
|
|
1076
|
+
// COR-233 gate: `<=` only when birthCanRetain AND this return also
|
|
1077
|
+
// matches the birth filter; raw (unaligned) birth time.
|
|
1078
|
+
const matchesFirst = retentionSideMatches(e, cohortEvent, cohortWhere);
|
|
1079
|
+
const passes = (birthCanRetain && matchesFirst) ? (birthMs <= evMs) : (birthMs < evMs);
|
|
513
1080
|
if (!passes) continue;
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
1081
|
+
const bucket = Math.floor((evMs - alignedBirthMs) / unitMs);
|
|
1082
|
+
if (bucket < 0) continue;
|
|
1083
|
+
// Write horizon (fix-round nit 3): ARB bounds-checks every write
|
|
1084
|
+
// against the queried bucket count — `bucket_index <
|
|
1085
|
+
// time_duration_buckets_num_buckets` (retention_query.cpp:1264) —
|
|
1086
|
+
// so a return past the last queried bucket is never recorded in
|
|
1087
|
+
// ANY mode. Without this, carryBack ("active in any bucket ≥ N")
|
|
1088
|
+
// would let an out-of-horizon return retro-mark every queried row.
|
|
1089
|
+
if (bucket > maxQueriedBucket) continue;
|
|
1090
|
+
const marked = markSetFor(segmentedOnReturn ? (e[segmentBy] ?? 'unknown') : birthSeg);
|
|
1091
|
+
// consecutiveForward write gate (retention_query.cpp:1275-1287):
|
|
1092
|
+
// bucket N marks only if N−1 already marked (except N = 0). Returns
|
|
1093
|
+
// arrive time-ordered, so marks form the maximal consecutive prefix.
|
|
1094
|
+
if (unbounded === 'consecutiveForward' && bucket !== 0 && !marked.has(bucket - 1)) continue;
|
|
1095
|
+
marked.add(bucket);
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
if (segmentedOnReturn) {
|
|
1099
|
+
// Births are unsegmented under segmentOn 'return' — a user joins a
|
|
1100
|
+
// segment's cohort only via a qualifying return in that segment
|
|
1101
|
+
// (retention_query.cpp:1413-1419; birthCanRetain lets the birth
|
|
1102
|
+
// itself qualify when it matches the return side).
|
|
1103
|
+
for (const [seg, marked] of markedBySeg) {
|
|
1104
|
+
const sb = ensureSegment(seg);
|
|
1105
|
+
sb.users.add(uid);
|
|
1106
|
+
sb.markedByUser.set(uid, marked);
|
|
1107
|
+
}
|
|
1108
|
+
} else {
|
|
1109
|
+
const sb = ensureSegment(birthSeg);
|
|
1110
|
+
sb.users.add(uid);
|
|
1111
|
+
sb.markedByUser.set(uid, markedBySeg.get(birthSeg) || new Set());
|
|
517
1112
|
}
|
|
518
|
-
sb.returnBucketsByUser.set(uid, retBuckets);
|
|
519
1113
|
}
|
|
520
1114
|
|
|
521
1115
|
const out = [];
|
|
@@ -524,18 +1118,20 @@ function retention(events, { cohortEvent, returnEvent, dayBuckets = [1, 7, 14, 3
|
|
|
524
1118
|
for (const day of dayBuckets) {
|
|
525
1119
|
let retained = 0;
|
|
526
1120
|
for (const uid of sb.users) {
|
|
527
|
-
const
|
|
528
|
-
if (!
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
for (const b of
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
if (
|
|
1121
|
+
const marked = sb.markedByUser.get(uid);
|
|
1122
|
+
if (!marked || !marked.size) continue;
|
|
1123
|
+
let hit = false;
|
|
1124
|
+
if (unbounded === 'carryForward') {
|
|
1125
|
+
// Read-time carry (:1854-1868): active in ANY bucket ≤ N.
|
|
1126
|
+
for (const b of marked) { if (b <= day) { hit = true; break; } }
|
|
1127
|
+
} else if (unbounded === 'carryBack') {
|
|
1128
|
+
// Reverse-iteration carry (:274-278): active in ANY bucket ≥ N.
|
|
1129
|
+
for (const b of marked) { if (b >= day) { hit = true; break; } }
|
|
536
1130
|
} else {
|
|
537
|
-
|
|
1131
|
+
// 'none' + 'consecutiveForward' (write-gated) read membership.
|
|
1132
|
+
hit = marked.has(day);
|
|
538
1133
|
}
|
|
1134
|
+
if (hit) retained++;
|
|
539
1135
|
}
|
|
540
1136
|
out.push({
|
|
541
1137
|
day,
|
|
@@ -554,16 +1150,25 @@ function retention(events, { cohortEvent, returnEvent, dayBuckets = [1, 7, 14, 3
|
|
|
554
1150
|
//
|
|
555
1151
|
// Reference: backend/arb/reader/queries/session_query.cpp.
|
|
556
1152
|
//
|
|
557
|
-
// Mixpanel computes sessions at
|
|
558
|
-
//
|
|
559
|
-
//
|
|
560
|
-
// `
|
|
561
|
-
//
|
|
1153
|
+
// Mixpanel computes sessions at QUERY TIME (30-min gap default + 24h max +
|
|
1154
|
+
// UTC day-boundary split) — sessions are never stored. v1.6.0 (P1.7.2)
|
|
1155
|
+
// aligns: `source: 'derived'` (default) re-derives sessions from timestamps
|
|
1156
|
+
// via sessionize(); `source: 'stamped'` keeps the v1.5 behavior of trusting
|
|
1157
|
+
// generator-stamped `session_id` (events without one are excluded there).
|
|
1158
|
+
// Declared v1.6.0 behavior change: the default no longer reads session_id
|
|
1159
|
+
// at all, so unstamped events now participate.
|
|
1160
|
+
//
|
|
1161
|
+
// When any event carries a stamped session_id, every row also reports
|
|
1162
|
+
// `stampedDivergence`: the number of events whose stamped session BOUNDARY
|
|
1163
|
+
// disagrees with the derived one — event i (per user, time-sorted) diverges
|
|
1164
|
+
// when `stamped(i) === stamped(i-1)` and `derived(i) === derived(i-1)`
|
|
1165
|
+
// disagree. Pairs where either event lacks a stamp are skipped (no boundary
|
|
1166
|
+
// to compare). This makes the generator-vs-query-time seam measurable.
|
|
562
1167
|
//
|
|
563
1168
|
// Returns an array with one row per requested metric:
|
|
564
|
-
// [{ metric: 'count', avg, median, p90, total_sessions }]
|
|
565
|
-
// [{ metric: 'duration', avg_ms, median_ms, p90_ms, total_sessions }]
|
|
566
|
-
// [{ metric: 'eventsPerSession',avg, median, p90, total_sessions }]
|
|
1169
|
+
// [{ metric: 'count', avg, median, p90, total_sessions, source, stampedDivergence }]
|
|
1170
|
+
// [{ metric: 'duration', avg_ms, median_ms, p90_ms, total_sessions, source, stampedDivergence }]
|
|
1171
|
+
// [{ metric: 'eventsPerSession',avg, median, p90, total_sessions, source, stampedDivergence }]
|
|
567
1172
|
|
|
568
1173
|
// ── COUNT_DISTINCT(property) — v1.5.1 ─────────────────────────────────────
|
|
569
1174
|
//
|
|
@@ -582,37 +1187,508 @@ function distinctCount(events, { property, event, topN = 25 }) {
|
|
|
582
1187
|
return [result];
|
|
583
1188
|
}
|
|
584
1189
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
1190
|
+
// ── Event Breakdown (Insights: TOTAL events segmented by a property) ──
|
|
1191
|
+
//
|
|
1192
|
+
// The most common Mixpanel report shape: count of EVENTS per property value
|
|
1193
|
+
// (not per-user grouping — that's aggregatePerUser). Semantics from
|
|
1194
|
+
// normal_query.cpp ACTION_TYPE_FOR_EACH (:1718-1776) + coerce.js rulebooks:
|
|
1195
|
+
//
|
|
1196
|
+
// - List-valued property → the event counts once PER list item (an event
|
|
1197
|
+
// with ["a","b"] increments both segments; recursion in
|
|
1198
|
+
// query_record_result over list_cursor items).
|
|
1199
|
+
// - Empty list → literal segment "$empty_list" (normal_query.cpp:1762
|
|
1200
|
+
// value_create_string("$empty_list")).
|
|
1201
|
+
// - null AND undefined both → the "undefined" bucket (string-typecast
|
|
1202
|
+
// default, arb_selector.py:889-916; normal_query.cpp:1769-1773 routes
|
|
1203
|
+
// VALUE_TYPE_UNDEFINED|NULL to the undefined inner action).
|
|
1204
|
+
// - Segment identity is case-SENSITIVE and type-tagged (hash_value.c raw
|
|
1205
|
+
// XXH3 string hash + per-type tag mixins; cmp.c arb_strcmp) — "iOS" and
|
|
1206
|
+
// "ios" are separate rows; number 1 and string "1" are separate rows.
|
|
1207
|
+
// - Sort count desc, keep top `topN` (default 250 — normal_query.cpp
|
|
1208
|
+
// :1195-1197 "If no meaningful limit is supplied, set it to 250"), and
|
|
1209
|
+
// TRUNCATE: no "other" rollup bucket (query_set_top_results marks the top
|
|
1210
|
+
// N leaf results; the rest simply don't render — normal_query.cpp
|
|
1211
|
+
// :1865-1905).
|
|
1212
|
+
// - v1.6.0 (P1.7.3) `countType: 'sessions'`: an event counts at most once
|
|
1213
|
+
// per (user, session, segment) — counted iff the segment has no prior
|
|
1214
|
+
// counted event in that session, then stamp
|
|
1215
|
+
// (query_record_per_user_session_segment_state, normal_query.cpp
|
|
1216
|
+
// :1318-1352 — the per-segment state timestamp is compared against the
|
|
1217
|
+
// user's CURRENT session start and updated on count). Sessions are
|
|
1218
|
+
// derived via sessionize() from the FULL event stream: ARB updates
|
|
1219
|
+
// session state on every event and only gates result RECORDING on the
|
|
1220
|
+
// filter (:2271-2280). `sessionTimeoutMs`/`maxSessionMs` thread through.
|
|
1221
|
+
// In sessions mode, `total_users` counts users of COUNTED events only.
|
|
1222
|
+
// - `countType: 'unique'`: the per-segment metric is distinct resolved
|
|
1223
|
+
// users (COUNT_TYPE_UNIQUE — query_record_per_user_state keeps a
|
|
1224
|
+
// per-(segment) user-state container, normal_query.cpp:1300-1316):
|
|
1225
|
+
// `count` equals `total_users`, and the topN cut sorts on that metric,
|
|
1226
|
+
// so a many-events/few-users segment can rank below a few-events/
|
|
1227
|
+
// many-users one. Unrecognized countType values THROW (same
|
|
1228
|
+
// strict-option rule as retention keys — a typo must not silently
|
|
1229
|
+
// fall back to general counting).
|
|
1230
|
+
//
|
|
1231
|
+
// Returns rows: `{ value, count, total_users }` where total_users is the
|
|
1232
|
+
// distinct identity-resolved user count within the segment.
|
|
1233
|
+
|
|
1234
|
+
function eventBreakdown(events, { event, breakdownProperty, topN = 250, countType, sessionTimeoutMs, maxSessionMs, firstTimeOnly = false, identityMap }) {
|
|
1235
|
+
if (!breakdownProperty) throw new Error('eventBreakdown requires `breakdownProperty`');
|
|
1236
|
+
if (countType !== undefined && countType !== 'general' && countType !== 'unique' && countType !== 'sessions') {
|
|
1237
|
+
throw new Error(`eventBreakdown: countType must be "general", "unique", or "sessions", got "${countType}"`);
|
|
1238
|
+
}
|
|
1239
|
+
// Derive BEFORE any event filtering (normal_query.cpp:2271-2280 — session
|
|
1240
|
+
// state updates are unconditional; the name/firstTime filter only gates
|
|
1241
|
+
// recording). Safe under the timeBucket wrapper's per-bucket slices:
|
|
1242
|
+
// sessions never cross UTC midnight (daySplit, sessionize.js) and bucket
|
|
1243
|
+
// units cut at midnights, so slice-derived boundaries equal full-stream
|
|
1244
|
+
// boundaries (B5 invariant tests).
|
|
1245
|
+
let sessionOf = null;
|
|
1246
|
+
if (countType === 'sessions') {
|
|
1247
|
+
const { sessions } = sessionize(events, {
|
|
1248
|
+
timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
|
|
1249
|
+
maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
|
|
1250
|
+
identityMap,
|
|
1251
|
+
});
|
|
1252
|
+
sessionOf = new Map();
|
|
1253
|
+
sessions.forEach((s, i) => { for (const e of s.events) sessionOf.set(e, i); });
|
|
1254
|
+
}
|
|
1255
|
+
if (firstTimeOnly) {
|
|
1256
|
+
// P1.4: restrict to each user's first-ever occurrence of `event`
|
|
1257
|
+
// before segmenting (the $nth_time_event rewrite — first-time.js).
|
|
1258
|
+
events = filterFirstTimeEver(events, { event, identityMap });
|
|
1259
|
+
}
|
|
1260
|
+
const segments = new Map(); // type-tagged segment key → { value, count, users, sessions }
|
|
1261
|
+
const record = (rawValue, e) => {
|
|
1262
|
+
const key = breakdownSegmentKey(rawValue);
|
|
1263
|
+
let seg = segments.get(key);
|
|
1264
|
+
if (!seg) {
|
|
1265
|
+
seg = { value: coerceToBreakdownKey(rawValue), count: 0, users: new Set(), sessions: sessionOf ? new Set() : null };
|
|
1266
|
+
segments.set(key, seg);
|
|
1267
|
+
}
|
|
1268
|
+
if (sessionOf) {
|
|
1269
|
+
const si = sessionOf.get(e);
|
|
1270
|
+
if (si === undefined) return; // no derived session (unresolvable id / bad time)
|
|
1271
|
+
if (seg.sessions.has(si)) return; // already counted this session for this segment
|
|
1272
|
+
seg.sessions.add(si); // stamp (normal_query.cpp:1330-1339)
|
|
1273
|
+
}
|
|
1274
|
+
seg.count++;
|
|
1275
|
+
seg.users.add(userIdOf(e, identityMap));
|
|
1276
|
+
};
|
|
1277
|
+
for (const e of events) {
|
|
1278
|
+
if (!e || (event && e.event !== event)) continue;
|
|
1279
|
+
const v = e[breakdownProperty];
|
|
1280
|
+
if (Array.isArray(v)) {
|
|
1281
|
+
if (v.length === 0) record('$empty_list', e);
|
|
1282
|
+
else for (const item of v) record(item, e);
|
|
1283
|
+
} else {
|
|
1284
|
+
record(v, e);
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
const unique = countType === 'unique';
|
|
1288
|
+
return [...segments.values()]
|
|
1289
|
+
.map(s => ({ value: s.value, count: unique ? s.users.size : s.count, total_users: s.users.size }))
|
|
1290
|
+
.sort((a, b) => b.count - a.count)
|
|
1291
|
+
.slice(0, topN);
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// ── Uniques (Insights: unique users per interval / XAU / cumulative) ──
|
|
1295
|
+
//
|
|
1296
|
+
// COUNT_TYPE_UNIQUE semantics from normal_query.cpp:
|
|
1297
|
+
// - Per-interval dedup is INDEPENDENT per bucket (query_record_per_user_state
|
|
1298
|
+
// keeps a per-(segment, interval) user-state container — :1300-1316): a user
|
|
1299
|
+
// active on 3 days counts 3 in a daily series. `unit: 'range'` is the same
|
|
1300
|
+
// machinery over a single interval (segmentation_arb.py:933 "total uniques").
|
|
1301
|
+
// - Rolling look-back window (WAU/MAU — window_length ≠ 0 branch, :1797-1830):
|
|
1302
|
+
// an event contributes to every interval whose decision post (interval END
|
|
1303
|
+
// for look-back windows) lies within [event_time, event_time + W], i.e. an
|
|
1304
|
+
// event on day E lands the user in daily buckets [E, E + W − 1]. XAU is a
|
|
1305
|
+
// look-back window over daily intervals — NOT a calendar week/month.
|
|
1306
|
+
// - Cumulative (accumulate_uniques_result, :1834-1863): each interval's count
|
|
1307
|
+
// becomes the size of the running distinct-id set through that interval.
|
|
1308
|
+
// - Events whose resolved user id is EMPTY are skipped (normal_query.cpp
|
|
1309
|
+
// :2200-2208 — distinct_id[0] == '\0' → return 0).
|
|
1310
|
+
// - v1.6.0 (P1.7.3) `countType: 'sessions'`: the bucket value is the count
|
|
1311
|
+
// of distinct (user, session) pairs active in the bucket — NOT distinct
|
|
1312
|
+
// users (per-interval result_values each own a user_session_states
|
|
1313
|
+
// container, normal_query.cpp:397-417; recording dedups per session via
|
|
1314
|
+
// query_record_per_user_session_segment_state, :1318-1352, dispatched at
|
|
1315
|
+
// :1462-1464). Sessions derive via sessionize() from the FULL stream
|
|
1316
|
+
// (state updates are unconditional, :2271-2280). Composes with
|
|
1317
|
+
// `rollingWindow` (the window branch at :1797-1830 routes through the
|
|
1318
|
+
// same count-type dispatch) but NOT `cumulative` — ARB expresses
|
|
1319
|
+
// cumulative only as the distinct COUNT_TYPE_CUMULATIVE_UNIQUE enum
|
|
1320
|
+
// (accumulate_uniques_result gate at :1860), so there is no cumulative
|
|
1321
|
+
// sessions count.
|
|
1322
|
+
//
|
|
1323
|
+
// Returns rows: `{ period, uniques }` sorted by period.
|
|
1324
|
+
|
|
1325
|
+
function uniques(events, {
|
|
1326
|
+
event,
|
|
1327
|
+
where,
|
|
1328
|
+
unit = 'day',
|
|
1329
|
+
rollingWindow,
|
|
1330
|
+
cumulative = false,
|
|
1331
|
+
countType,
|
|
1332
|
+
sessionTimeoutMs,
|
|
1333
|
+
maxSessionMs,
|
|
1334
|
+
firstTimeOnly = false,
|
|
1335
|
+
identityMap,
|
|
1336
|
+
}) {
|
|
1337
|
+
let sessionOf = null;
|
|
1338
|
+
if (countType === 'sessions') {
|
|
1339
|
+
if (cumulative) {
|
|
1340
|
+
throw new Error('uniques: cumulative sessions is not supported (ARB has no cumulative sessions count type — normal_query.cpp:1860)');
|
|
1341
|
+
}
|
|
1342
|
+
// Derive BEFORE the event/where filter (normal_query.cpp:2271-2280).
|
|
1343
|
+
const { sessions } = sessionize(events, {
|
|
1344
|
+
timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
|
|
1345
|
+
maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
|
|
1346
|
+
identityMap,
|
|
1347
|
+
});
|
|
1348
|
+
sessionOf = new Map();
|
|
1349
|
+
sessions.forEach((s, i) => { for (const e of s.events) sessionOf.set(e, i); });
|
|
1350
|
+
}
|
|
1351
|
+
if (rollingWindow != null && cumulative) {
|
|
1352
|
+
throw new Error('uniques: rollingWindow and cumulative are mutually exclusive');
|
|
1353
|
+
}
|
|
1354
|
+
if (rollingWindow != null && (!Number.isInteger(rollingWindow) || rollingWindow < 1)) {
|
|
1355
|
+
throw new Error('uniques: rollingWindow must be a positive integer');
|
|
1356
|
+
}
|
|
1357
|
+
if (!['day', 'week', 'month', 'range'].includes(unit)) {
|
|
1358
|
+
throw new Error(`uniques: unknown unit "${unit}"`);
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
if (firstTimeOnly) {
|
|
1362
|
+
// P1.4: `where` acts as the PRE-filter — it defines the universe the
|
|
1363
|
+
// per-user first is picked from. The rewritten query does not re-apply
|
|
1364
|
+
// it (see first-time.js header), so the loop below must not either.
|
|
1365
|
+
events = filterFirstTimeEver(events, { event, preWhere: where, identityMap });
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
// `key` is the per-bucket dedup identity: the resolved user id, or the
|
|
1369
|
+
// derived session index in sessions mode (a session belongs to one user,
|
|
1370
|
+
// so the index IS the (user, session) pair).
|
|
1371
|
+
const filtered = [];
|
|
1372
|
+
for (const e of events) {
|
|
1373
|
+
if (!e || (event && e.event !== event)) continue;
|
|
1374
|
+
if (!firstTimeOnly && !matchesWhere(e, where)) continue;
|
|
1375
|
+
const uid = userIdOf(e, identityMap);
|
|
1376
|
+
if (uid === undefined || uid === null || uid === '') continue; // :2200-2208
|
|
1377
|
+
const key = sessionOf ? sessionOf.get(e) : uid;
|
|
1378
|
+
if (key === undefined) continue; // sessions mode: no derived session (bad time)
|
|
1379
|
+
filtered.push({ e, key });
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
if (rollingWindow != null) {
|
|
1383
|
+
// Daily intervals only (unit forced to 'day'); W-day look-back per bucket.
|
|
1384
|
+
const W = rollingWindow;
|
|
1385
|
+
const DAY = 86400_000;
|
|
1386
|
+
let minDay = Infinity, maxDay = -Infinity;
|
|
1387
|
+
/** @type {Array<[number, string|number]>} */
|
|
1388
|
+
const eventDays = []; // [dayIndex, key]
|
|
1389
|
+
for (const { e, key } of filtered) {
|
|
1390
|
+
const ms = toMs(e.time);
|
|
1391
|
+
if (!Number.isFinite(ms)) continue;
|
|
1392
|
+
const d = Math.floor(ms / DAY);
|
|
1393
|
+
eventDays.push([d, key]);
|
|
1394
|
+
if (d < minDay) minDay = d;
|
|
1395
|
+
if (d > maxDay) maxDay = d;
|
|
1396
|
+
}
|
|
1397
|
+
if (!eventDays.length) return [];
|
|
1398
|
+
const perBucket = new Map(); // dayIndex → Set<key>
|
|
1399
|
+
for (let d = minDay; d <= maxDay; d++) perBucket.set(d, new Set());
|
|
1400
|
+
for (const [d, key] of eventDays) {
|
|
1401
|
+
// Event on day E → buckets [E, E + W − 1], clamped to the observed range.
|
|
1402
|
+
// Exact-midnight boundary (fix-round nit 6, REFUTED): ARB's decision
|
|
1403
|
+
// check is end-post inclusive (normal_query.cpp:1825 `post >= time_min`,
|
|
1404
|
+
// and end(E−1) == an exact-midnight event_time), which would admit day
|
|
1405
|
+
// E−1 — but the interval scan STARTS at
|
|
1406
|
+
// `interval_min = uniform_intervals_for_relaxed(time_min)` (:1801), and
|
|
1407
|
+
// a fencepost timestamp maps to the LATER interval (integer division,
|
|
1408
|
+
// uniform_intervals.h:57), so day E−1 is never evaluated. Upper edge:
|
|
1409
|
+
// end(E+W−1) == time_max for a midnight event → inclusive → included.
|
|
1410
|
+
// Net: [E, E+W−1] for every event, midnight or not — exactly this loop.
|
|
1411
|
+
const hi = Math.min(d + W - 1, maxDay);
|
|
1412
|
+
for (let b = d; b <= hi; b++) perBucket.get(b).add(key);
|
|
1413
|
+
}
|
|
1414
|
+
return [...perBucket.entries()].map(([d, set]) => ({
|
|
1415
|
+
period: new Date(d * DAY).toISOString().slice(0, 10),
|
|
1416
|
+
uniques: set.size,
|
|
1417
|
+
}));
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
if (unit === 'range') {
|
|
1421
|
+
const all = new Set(filtered.map(({ key }) => key));
|
|
1422
|
+
return [{ period: 'range', uniques: all.size }];
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
const buckets = partitionByTimeBucket(filtered.map(({ e }) => e), /** @type {*} */ (unit));
|
|
1426
|
+
// Re-resolve key per event inside each bucket via a lookup built above —
|
|
1427
|
+
// avoid resolving twice by mapping event object → key.
|
|
1428
|
+
const keyOf = new Map(filtered.map(({ e, key }) => [e, key]));
|
|
1429
|
+
const running = new Set();
|
|
1430
|
+
return buckets.map(({ period, events: evs }) => {
|
|
1431
|
+
const set = new Set();
|
|
1432
|
+
for (const e of evs) set.add(keyOf.get(e));
|
|
1433
|
+
if (cumulative) {
|
|
1434
|
+
for (const u of set) running.add(u);
|
|
1435
|
+
return { period, uniques: running.size };
|
|
1436
|
+
}
|
|
1437
|
+
return { period, uniques: set.size };
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
// ── Lifecycle (P1.8 — the LCA board template classification) ──
|
|
1442
|
+
//
|
|
1443
|
+
// Mixpanel has NO engine lifecycle query (query_type.cpp:8-30 enumerates
|
|
1444
|
+
// every query type — nothing lifecycle-shaped). "Lifecycle" ships as the
|
|
1445
|
+
// Lifecycle Cohort Analysis board template: Insights uniques filtered by
|
|
1446
|
+
// four behavioral cohorts on a Value Moment event, in 7- and 30-day period
|
|
1447
|
+
// variants (iron/common/report/dashboards/types.ts:367-390 —
|
|
1448
|
+
// LifecycleCohortAnalysisTemplateFields.VALUE_MOMENT is the one required
|
|
1449
|
+
// template field). The canonical in-source cohort definition is
|
|
1450
|
+
// weeklyResurrectedUserBookmark
|
|
1451
|
+
// (iron/common/widgets/profile-summary/bookmark_templates.ts:179-320):
|
|
1452
|
+
// resurrected = value-moment count AtLeast 1 in the last 30d AND EqualTo 0
|
|
1453
|
+
// in 60→30d ago AND AtLeast 1 in 90→60d ago. Generalized to period P:
|
|
1454
|
+
// active in T, inactive in T−1, active in some period before T−1.
|
|
1455
|
+
//
|
|
1456
|
+
// ARB-DIVERGENCE (Resurrected lookback — fix-round Q6, KEEP): the bookmark's
|
|
1457
|
+
// third clause looks back exactly ONE period past the dormancy gap (AtLeast 1
|
|
1458
|
+
// in 90→60d ago = active in T−2 specifically). This emulator accepts activity
|
|
1459
|
+
// in ANY period before T−1 — growth accounting: every active user is exactly
|
|
1460
|
+
// one of New/Retained/Resurrected, so new + retained + resurrected = period
|
|
1461
|
+
// uniques and the classes tile the actives with no unclassified remainder.
|
|
1462
|
+
// Under the literal bookmark, a returner whose last activity predates T−2
|
|
1463
|
+
// falls out of ALL four cohorts (the LCA board undercounts long-dormant
|
|
1464
|
+
// returners by construction). Dungeon stories assert on the growth-accounting
|
|
1465
|
+
// partition; anyone reconciling against a real LCA board should expect
|
|
1466
|
+
// `resurrected` here ≥ the board's whenever dormancy gaps exceed one period.
|
|
1467
|
+
//
|
|
1468
|
+
// DECLARED DIVERGENCE (tiled vs rolling): the real template's cohort
|
|
1469
|
+
// windows are rolling, re-anchored as-of each charting interval
|
|
1470
|
+
// (api/version_2_0/insights/params.py:1396-1403
|
|
1471
|
+
// generate_behaviors_as_of_project_times_for_cohorts →
|
|
1472
|
+
// behaviors/count.py:213-300 behavior_as_of_project_times;
|
|
1473
|
+
// cohort_count_query.cpp:48-59 — per-interval user-state containers with an
|
|
1474
|
+
// associated_time). Tiling fixed periods back from the dataset's LAST EVENT
|
|
1475
|
+
// DAY is the deterministic equivalent for generated fixture data: identical
|
|
1476
|
+
// classification rules, stable period edges. Rows span from the first
|
|
1477
|
+
// value-moment event day through the dataset's last event day; the earliest
|
|
1478
|
+
// tile may be partial, and its actives are all New by dataset-boundedness
|
|
1479
|
+
// (the real template's ~5y first-time lookback always covers a dungeon
|
|
1480
|
+
// window — see first-time.js header).
|
|
1481
|
+
//
|
|
1482
|
+
// The dormancy test is an EqualTo-0 filter (bookmark_templates.ts) — ONE
|
|
1483
|
+
// stray value-moment event inside a would-be dormancy window reclassifies
|
|
1484
|
+
// the user (resurrected → retained). Gap discipline is therefore a
|
|
1485
|
+
// story-authoring requirement (P2.3 atom).
|
|
1486
|
+
function lifecycle(events, { valueMomentEvent, periodDays = 7, identityMap }) {
|
|
1487
|
+
if (!valueMomentEvent) throw new Error('lifecycle requires valueMomentEvent');
|
|
1488
|
+
if (periodDays !== 7 && periodDays !== 30) {
|
|
1489
|
+
throw new Error(`lifecycle: periodDays must be 7 or 30 (the LCA template variants), got ${periodDays}`);
|
|
1490
|
+
}
|
|
1491
|
+
const DAY = 86400_000;
|
|
1492
|
+
const P = periodDays;
|
|
1493
|
+
|
|
1494
|
+
// Anchor period edges on the FULL stream's last event day — the dataset
|
|
1495
|
+
// boundary, not the value-moment subset: a value moment that stops firing
|
|
1496
|
+
// must still be reported dormant through the end of the data.
|
|
1497
|
+
let lastDay = -Infinity;
|
|
1498
|
+
for (const e of events) {
|
|
1499
|
+
const ms = toMs(e && e.time);
|
|
1500
|
+
if (Number.isFinite(ms)) {
|
|
1501
|
+
const d = Math.floor(ms / DAY);
|
|
1502
|
+
if (d > lastDay) lastDay = d;
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
// "Active" = did the value-moment event; other events never qualify
|
|
1507
|
+
// (the template's cohorts all filter on the one Value Moment —
|
|
1508
|
+
// bookmark_templates.ts custom-property behaviors).
|
|
1509
|
+
const vm = [];
|
|
1510
|
+
let firstDay = Infinity;
|
|
1511
|
+
for (const e of events) {
|
|
1512
|
+
if (!e || e.event !== valueMomentEvent) continue;
|
|
1513
|
+
const uid = userIdOf(e, identityMap);
|
|
1514
|
+
if (uid === undefined || uid === null || uid === '') continue; // normal_query.cpp:2200-2208
|
|
1515
|
+
const ms = toMs(e.time);
|
|
1516
|
+
if (!Number.isFinite(ms)) continue;
|
|
1517
|
+
const day = Math.floor(ms / DAY);
|
|
1518
|
+
vm.push({ uid, day });
|
|
1519
|
+
if (day < firstDay) firstDay = day;
|
|
1520
|
+
}
|
|
1521
|
+
if (!vm.length) return [];
|
|
1522
|
+
|
|
1523
|
+
// Period i (ascending) covers day indexes (endDay(i) − P, endDay(i)],
|
|
1524
|
+
// endDay(i) = lastDay − (numPeriods − 1 − i) · P.
|
|
1525
|
+
const numPeriods = Math.ceil((lastDay - firstDay + 1) / P);
|
|
1526
|
+
const periodOf = (day) => numPeriods - 1 - Math.floor((lastDay - day) / P);
|
|
1527
|
+
|
|
1528
|
+
const activity = new Map(); // uid → Set<periodIdx>
|
|
1529
|
+
for (const { uid, day } of vm) {
|
|
1530
|
+
let set = activity.get(uid);
|
|
1531
|
+
if (!set) activity.set(uid, set = new Set());
|
|
1532
|
+
set.add(periodOf(day));
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
// New = first-ever value moment falls in T. LCA's New cohort rides the
|
|
1536
|
+
// $nth_time_event rewrite, so reuse P1.4's machinery (first-time.js);
|
|
1537
|
+
// its both-pass timestamp-tie edge is harmless here — same user, same
|
|
1538
|
+
// timestamp, same period.
|
|
1539
|
+
const firstPeriod = new Map(); // uid → periodIdx
|
|
1540
|
+
for (const e of filterFirstTimeEver(events, { event: valueMomentEvent, identityMap })) {
|
|
1541
|
+
const uid = userIdOf(e, identityMap);
|
|
1542
|
+
const ms = toMs(e.time);
|
|
1543
|
+
if (uid === undefined || uid === null || uid === '' || !Number.isFinite(ms)) continue;
|
|
1544
|
+
if (!firstPeriod.has(uid)) firstPeriod.set(uid, periodOf(Math.floor(ms / DAY)));
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
const rows = [];
|
|
1548
|
+
for (let i = 0; i < numPeriods; i++) {
|
|
1549
|
+
const endDay = lastDay - (numPeriods - 1 - i) * P;
|
|
1550
|
+
const row = {
|
|
1551
|
+
period: new Date(endDay * DAY).toISOString().slice(0, 10),
|
|
1552
|
+
new: 0,
|
|
1553
|
+
retained: 0,
|
|
1554
|
+
resurrected: 0,
|
|
1555
|
+
dormant: 0,
|
|
1556
|
+
};
|
|
1557
|
+
for (const [uid, set] of activity) {
|
|
1558
|
+
const active = set.has(i);
|
|
1559
|
+
const activePrev = i > 0 && set.has(i - 1);
|
|
1560
|
+
if (active) {
|
|
1561
|
+
// New takes precedence: a first-ever event can't retain or
|
|
1562
|
+
// resurrect (no prior activity by definition), so the branch
|
|
1563
|
+
// order below is belt-and-braces — but explicit per spec.
|
|
1564
|
+
if (firstPeriod.get(uid) === i) {
|
|
1565
|
+
row.new++;
|
|
1566
|
+
} else if (activePrev) {
|
|
1567
|
+
row.retained++;
|
|
1568
|
+
} else {
|
|
1569
|
+
// Resurrected needs activity in some period BEFORE the
|
|
1570
|
+
// (inactive) T−1 — ANY prior period, not the bookmark's
|
|
1571
|
+
// exact T−2 (see ARB-DIVERGENCE header: growth
|
|
1572
|
+
// accounting keeps the partition exhaustive). Non-first
|
|
1573
|
+
// active with inactive T−1 always has one, so this scan
|
|
1574
|
+
// can't miss — kept as a real check rather than an
|
|
1575
|
+
// assumption.
|
|
1576
|
+
for (const j of set) {
|
|
1577
|
+
if (j < i - 1) { row.resurrected++; break; }
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
} else if (activePrev) {
|
|
1581
|
+
// EqualTo-0 in T, AtLeast-1 in T−1 — dormant counts exactly
|
|
1582
|
+
// one period, then the user drops out of every class until
|
|
1583
|
+
// (unless) they resurrect. Reported positive; charting it
|
|
1584
|
+
// negative is a frontend concern.
|
|
1585
|
+
row.dormant++;
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
rows.push(row);
|
|
1589
|
+
}
|
|
1590
|
+
return rows;
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
// ── Top Paths (P1.9 — the Flows report) ──
|
|
1594
|
+
//
|
|
1595
|
+
// Thin dispatch over the flows module: extractFlows runs the per-user ARB
|
|
1596
|
+
// state machine (anchors, buffers, hidden/visible, collapse, count types),
|
|
1597
|
+
// aggregateFlows builds + prunes the prefix tree and shapes the list/sankey
|
|
1598
|
+
// output. All ARB citations live in flows.js.
|
|
1599
|
+
function topPaths(events, {
|
|
1600
|
+
anchors, forward, reverse, countType, hiddenEvents, visibleEvents,
|
|
1601
|
+
collapseRepeated, breakdownProperty, sessionTimeoutMs, maxSessionMs,
|
|
1602
|
+
identityMap, output, cardinalityThreshold, listSortPriority,
|
|
1603
|
+
}) {
|
|
1604
|
+
const flows = extractFlows(events, {
|
|
1605
|
+
anchors, forward, reverse, countType, hiddenEvents, visibleEvents,
|
|
1606
|
+
collapseRepeated, breakdownProperty, sessionTimeoutMs, maxSessionMs,
|
|
1607
|
+
identityMap,
|
|
1608
|
+
});
|
|
1609
|
+
const opts = { output, cardinalityThreshold };
|
|
1610
|
+
if (listSortPriority !== undefined) /** @type {*} */ (opts).listSortPriority = listSortPriority;
|
|
1611
|
+
return aggregateFlows(flows, opts);
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
function sessionMetrics(events, { event, metrics = ['count', 'duration', 'eventsPerSession'], source = 'derived', sessionTimeoutMs, maxSessionMs, identityMap }) {
|
|
1615
|
+
if (source !== 'derived' && source !== 'stamped') {
|
|
1616
|
+
throw new Error(`sessionMetrics source must be 'derived' or 'stamped', got '${source}'`);
|
|
1617
|
+
}
|
|
1618
|
+
// Query-time derivation runs in BOTH modes: it is the metric source when
|
|
1619
|
+
// source==='derived', and the divergence reference when source==='stamped'.
|
|
1620
|
+
const { sessions } = sessionize(events, {
|
|
1621
|
+
timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
|
|
1622
|
+
maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
|
|
1623
|
+
identityMap,
|
|
1624
|
+
});
|
|
1625
|
+
|
|
1626
|
+
// stampedDivergence — boundary disagreements between consecutive stamped
|
|
1627
|
+
// events (see block comment above). Derived membership is by object
|
|
1628
|
+
// reference: sessionize partitions each user's events, so `sidOf` is total
|
|
1629
|
+
// over resolvable events.
|
|
1630
|
+
const hasStamps = events.some(e => e && e.session_id != null);
|
|
1631
|
+
let stampedDivergence = null;
|
|
1632
|
+
if (hasStamps) {
|
|
1633
|
+
const sidOf = new Map();
|
|
1634
|
+
sessions.forEach((s, i) => { for (const e of s.events) sidOf.set(e, i); });
|
|
1635
|
+
stampedDivergence = 0;
|
|
1636
|
+
for (const [, evs] of groupByUser(events, identityMap)) {
|
|
1637
|
+
const sorted = sortByTime(evs);
|
|
1638
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1639
|
+
const a = sorted[i - 1], b = sorted[i];
|
|
1640
|
+
if (a.session_id == null || b.session_id == null) continue;
|
|
1641
|
+
const stampedSame = String(a.session_id) === String(b.session_id);
|
|
1642
|
+
const derivedSame = sidOf.get(a) !== undefined && sidOf.get(a) === sidOf.get(b);
|
|
1643
|
+
if (stampedSame !== derivedSame) stampedDivergence++;
|
|
603
1644
|
}
|
|
604
1645
|
}
|
|
605
|
-
if (buckets.size) sessionsByUser.set(uid, buckets);
|
|
606
1646
|
}
|
|
1647
|
+
|
|
607
1648
|
const allSessions = []; // { duration_ms, event_count }
|
|
608
1649
|
const sessionCountsPerUser = [];
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
1650
|
+
if (source === 'derived') {
|
|
1651
|
+
// Optional event filter: only sessions containing this event qualify
|
|
1652
|
+
// (session.events holds references to the original records).
|
|
1653
|
+
const qualifying = event
|
|
1654
|
+
? sessions.filter(s => s.events.some(e => e && e.event === event))
|
|
1655
|
+
: sessions;
|
|
1656
|
+
const perUser = new Map();
|
|
1657
|
+
for (const s of qualifying) {
|
|
1658
|
+
perUser.set(s.userId, (perUser.get(s.userId) || 0) + 1);
|
|
1659
|
+
// endMs is stamped at the session's LAST event (idle tail excluded),
|
|
1660
|
+
// so endMs - startMs matches $duration_s at ms resolution.
|
|
1661
|
+
allSessions.push({ duration_ms: s.endMs - s.startMs, event_count: s.event_count });
|
|
1662
|
+
}
|
|
1663
|
+
sessionCountsPerUser.push(...perUser.values());
|
|
1664
|
+
} else {
|
|
1665
|
+
// v1.5 stamped path: one session bucket per (user, session_id); events
|
|
1666
|
+
// without session_id are excluded.
|
|
1667
|
+
const userEvents = groupByUser(events, identityMap);
|
|
1668
|
+
const sessionsByUser = new Map();
|
|
1669
|
+
for (const [uid, evs] of userEvents) {
|
|
1670
|
+
const buckets = new Map();
|
|
1671
|
+
for (const ev of evs) {
|
|
1672
|
+
if (ev.session_id == null) continue;
|
|
1673
|
+
const sid = String(ev.session_id);
|
|
1674
|
+
if (!buckets.has(sid)) buckets.set(sid, []);
|
|
1675
|
+
buckets.get(sid).push(ev);
|
|
1676
|
+
}
|
|
1677
|
+
if (event) {
|
|
1678
|
+
for (const [sid, evs2] of [...buckets]) {
|
|
1679
|
+
if (!evs2.some(e => e.event === event)) buckets.delete(sid);
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
if (buckets.size) sessionsByUser.set(uid, buckets);
|
|
1683
|
+
}
|
|
1684
|
+
for (const [, buckets] of sessionsByUser) {
|
|
1685
|
+
sessionCountsPerUser.push(buckets.size);
|
|
1686
|
+
for (const [, evs] of buckets) {
|
|
1687
|
+
const sorted = sortByTime(evs);
|
|
1688
|
+
const start = toMs(sorted[0].time);
|
|
1689
|
+
const end = toMs(sorted[sorted.length - 1].time);
|
|
1690
|
+
allSessions.push({ duration_ms: end - start, event_count: sorted.length });
|
|
1691
|
+
}
|
|
616
1692
|
}
|
|
617
1693
|
}
|
|
618
1694
|
const out = [];
|
|
@@ -624,6 +1700,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
|
|
|
624
1700
|
median: median(sessionCountsPerUser),
|
|
625
1701
|
p90: percentile(sessionCountsPerUser, 0.9),
|
|
626
1702
|
total_sessions: allSessions.length,
|
|
1703
|
+
source,
|
|
1704
|
+
stampedDivergence,
|
|
627
1705
|
});
|
|
628
1706
|
}
|
|
629
1707
|
if (requested.has('duration')) {
|
|
@@ -634,6 +1712,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
|
|
|
634
1712
|
median_ms: median(durations),
|
|
635
1713
|
p90_ms: percentile(durations, 0.9),
|
|
636
1714
|
total_sessions: allSessions.length,
|
|
1715
|
+
source,
|
|
1716
|
+
stampedDivergence,
|
|
637
1717
|
});
|
|
638
1718
|
}
|
|
639
1719
|
if (requested.has('eventsPerSession')) {
|
|
@@ -644,6 +1724,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
|
|
|
644
1724
|
median: median(eventCounts),
|
|
645
1725
|
p90: percentile(eventCounts, 0.9),
|
|
646
1726
|
total_sessions: allSessions.length,
|
|
1727
|
+
source,
|
|
1728
|
+
stampedDivergence,
|
|
647
1729
|
});
|
|
648
1730
|
}
|
|
649
1731
|
return out;
|