@ak--47/dungeon-master 1.5.3 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +21 -12
  2. package/.claude/skills/create-dungeon/SKILL.md +65 -5
  3. package/.claude/skills/create-project/SKILL.md +96 -0
  4. package/.claude/skills/create-project/provision.mjs +336 -0
  5. package/.claude/skills/verify-dungeon/SKILL.md +56 -25
  6. package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
  7. package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
  8. package/.claude/skills/write-hooks/SKILL.md +156 -18
  9. package/CHANGELOG.md +211 -0
  10. package/HOOKS.md +566 -112
  11. package/README.md +28 -9
  12. package/docs/guides/1.3.0-upgrade-guide.md +262 -0
  13. package/docs/guides/1.3.1-upgrade-guide.md +328 -0
  14. package/docs/guides/1.4.0-upgrade-guide.md +130 -0
  15. package/docs/guides/1.4.1-upgrade-guide.md +125 -0
  16. package/docs/guides/1.4.3-upgrade-guide.md +200 -0
  17. package/docs/guides/1.4.4-upgrade-guide.md +95 -0
  18. package/docs/guides/1.5.0-upgrade-guide.md +426 -0
  19. package/docs/guides/1.5.1-upgrade-guide.md +178 -0
  20. package/docs/guides/1.5.3-upgrade-guide.md +117 -0
  21. package/docs/guides/1.6.0-upgrade-guide.md +204 -0
  22. package/dungeons/technical/stories-verify.js +152 -0
  23. package/dungeons/vertical/README.md +154 -0
  24. package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
  25. package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
  26. package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
  27. package/dungeons/vertical/community/community.js +1589 -0
  28. package/dungeons/vertical/community/community.sql +216 -0
  29. package/dungeons/vertical/community/community.verify.mjs +72 -0
  30. package/dungeons/vertical/crypto/crypto.js +1483 -0
  31. package/dungeons/vertical/crypto/crypto.sql +194 -0
  32. package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
  33. package/dungeons/vertical/dating/dating.js +1540 -0
  34. package/dungeons/vertical/dating/dating.sql +288 -0
  35. package/dungeons/vertical/dating/dating.verify.mjs +72 -0
  36. package/dungeons/vertical/devtools/devtools.js +1524 -0
  37. package/dungeons/vertical/devtools/devtools.sql +161 -0
  38. package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
  39. package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
  40. package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
  41. package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
  42. package/dungeons/vertical/education/education.js +1763 -0
  43. package/dungeons/vertical/education/education.sql +243 -0
  44. package/dungeons/vertical/education/education.verify.mjs +75 -0
  45. package/dungeons/vertical/fintech/fintech.js +1410 -0
  46. package/dungeons/vertical/fintech/fintech.sql +175 -0
  47. package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
  48. package/dungeons/vertical/fitness/fitness.js +1495 -0
  49. package/dungeons/vertical/fitness/fitness.sql +206 -0
  50. package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
  51. package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
  52. package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
  53. package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
  54. package/dungeons/vertical/gaming/gaming.js +1741 -0
  55. package/dungeons/vertical/gaming/gaming.sql +249 -0
  56. package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
  57. package/dungeons/vertical/healthcare/healthcare.js +1488 -0
  58. package/dungeons/vertical/healthcare/healthcare.sql +193 -0
  59. package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
  60. package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
  61. package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
  62. package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
  63. package/dungeons/vertical/logistics/logistics.js +1541 -0
  64. package/dungeons/vertical/logistics/logistics.sql +205 -0
  65. package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
  66. package/dungeons/vertical/marketplace/marketplace.js +1438 -0
  67. package/dungeons/vertical/marketplace/marketplace.sql +148 -0
  68. package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
  69. package/dungeons/vertical/{media.js → media/media.js} +557 -58
  70. package/dungeons/vertical/media/media.sql +173 -0
  71. package/dungeons/vertical/media/media.verify.mjs +74 -0
  72. package/dungeons/vertical/real-estate/real-estate.js +1629 -0
  73. package/dungeons/vertical/real-estate/real-estate.sql +167 -0
  74. package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
  75. package/dungeons/vertical/sass/sass.js +1733 -0
  76. package/dungeons/vertical/sass/sass.sql +173 -0
  77. package/dungeons/vertical/sass/sass.verify.mjs +74 -0
  78. package/dungeons/vertical/social/social.js +1456 -0
  79. package/dungeons/vertical/social/social.sql +160 -0
  80. package/dungeons/vertical/social/social.verify.mjs +76 -0
  81. package/dungeons/vertical/streaming/streaming.js +677 -0
  82. package/dungeons/vertical/streaming/streaming.sql +101 -0
  83. package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
  84. package/dungeons/vertical/support-desk/support-desk.js +741 -0
  85. package/dungeons/vertical/support-desk/support-desk.sql +137 -0
  86. package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
  87. package/dungeons/vertical/travel/travel.js +1603 -0
  88. package/dungeons/vertical/travel/travel.sql +236 -0
  89. package/dungeons/vertical/travel/travel.verify.mjs +72 -0
  90. package/index.js +2 -2
  91. package/lib/core/config-validator.js +37 -2
  92. package/lib/core/context.js +4 -2
  93. package/lib/core/dungeon-loader.js +20 -15
  94. package/lib/core/dungeon-to-json.js +22 -4
  95. package/lib/core/storage.js +3 -3
  96. package/lib/generators/funnels.js +36 -9
  97. package/lib/hook-helpers/cohort.js +53 -0
  98. package/lib/hook-helpers/index.js +8 -0
  99. package/lib/hook-helpers/mutate.js +1 -1
  100. package/lib/hook-helpers/shape.js +319 -0
  101. package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
  102. package/lib/hook-patterns/attributed-by-source.js +60 -58
  103. package/lib/hook-patterns/frequency-by-frequency.js +14 -4
  104. package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
  105. package/lib/hook-patterns/index.js +1 -1
  106. package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
  107. package/lib/orchestrators/user-loop.js +41 -0
  108. package/lib/templates/story-spec.schema.json +175 -0
  109. package/lib/utils/json-evaluator.js +1 -1
  110. package/lib/utils/utils.js +5 -4
  111. package/lib/verify/coerce.js +186 -0
  112. package/lib/verify/counting.js +270 -32
  113. package/lib/verify/emulate-breakdown.js +1285 -203
  114. package/lib/verify/first-time.js +89 -0
  115. package/lib/verify/flows.js +679 -0
  116. package/lib/verify/formula.js +259 -0
  117. package/lib/verify/funnel-engine.js +697 -113
  118. package/lib/verify/index.js +32 -4
  119. package/lib/verify/sessionize.js +275 -0
  120. package/lib/verify/story-runner.js +483 -0
  121. package/lib/verify/verify-dungeon.js +59 -37
  122. package/package.json +3 -2
  123. package/scripts/extract-dungeon-schema.mjs +2 -2
  124. package/scripts/run-many.mjs +2 -2
  125. package/scripts/verify-runner.mjs +2 -2
  126. package/scripts/verify-stories.mjs +244 -0
  127. package/types.d.ts +296 -30
  128. package/dungeons/vertical/ai-platform.js +0 -818
  129. package/dungeons/vertical/community.js +0 -966
  130. package/dungeons/vertical/crypto.js +0 -895
  131. package/dungeons/vertical/dating.js +0 -830
  132. package/dungeons/vertical/devtools.js +0 -1050
  133. package/dungeons/vertical/ecommerce.js +0 -811
  134. package/dungeons/vertical/education.js +0 -964
  135. package/dungeons/vertical/fintech.js +0 -946
  136. package/dungeons/vertical/fitness.js +0 -935
  137. package/dungeons/vertical/food-delivery.js +0 -934
  138. package/dungeons/vertical/gaming.js +0 -1120
  139. package/dungeons/vertical/healthcare.js +0 -945
  140. package/dungeons/vertical/insurance-application.js +0 -1039
  141. package/dungeons/vertical/logistics.js +0 -986
  142. package/dungeons/vertical/marketplace.js +0 -1019
  143. package/dungeons/vertical/real-estate.js +0 -872
  144. package/dungeons/vertical/sass.js +0 -995
  145. package/dungeons/vertical/social.js +0 -904
  146. package/dungeons/vertical/travel.js +0 -930
@@ -22,8 +22,12 @@
22
22
  */
23
23
 
24
24
  import { toMs } from '../hook-helpers/_internal.js';
25
- import { evaluateFunnel, evaluateAnyOrderCompletion } from './funnel-engine.js';
25
+ import { evaluateFunnel, evaluateFunnelHPC, evaluateAnyOrderCompletion } from './funnel-engine.js';
26
26
  import { buildIdentityMap, resolveUserId } from './identity.js';
27
+ import { coerceToBreakdownKey, breakdownSegmentKey, matchesWhere } from './coerce.js';
28
+ import { filterFirstTimeEver } from './first-time.js';
29
+ import { sessionize } from './sessionize.js';
30
+ import { extractFlows, aggregateFlows } from './flows.js';
27
31
  import {
28
32
  countDistinctPeriods,
29
33
  countDistinctValues,
@@ -31,69 +35,80 @@ import {
31
35
  nullAwareSum,
32
36
  nullAwareExtreme,
33
37
  partitionByTimeBucket,
38
+ bucketBoundsMs,
34
39
  } from './counting.js';
35
40
 
36
41
  /**
37
42
  * v1.5: pick the right per-user funnel evaluator based on the funnel's `order` mode.
43
+ * v1.6.0 (P1.6.6): orders whose scrambled region is CONTIGUOUS map exactly onto
44
+ * the engine's `{ anyOrder: [...] }` step blocks (history.cpp anchor/chunk
45
+ * greedy pass) — full Mixpanel semantics: conversion window, 2s rule,
46
+ * exclusions, reentry, and `anchorRange` all apply.
38
47
  *
39
- * - `sequential`, `interrupt`, `interrupted` → greedy single-pass (Mixpanel-aligned)
40
- * - `first-fixed` step-0 greedy + any-order on rest
41
- * - `last-fixed`, `outside-in`, `middle-fixed`, `first-and-last-fixed`
42
- * any-order completion (partial verification)
43
- * - `random` → any-order, informational only
48
+ * - `sequential`, `interrupt`, `interrupted` → greedy single-pass
49
+ * - `first-fixed` `[s0, { anyOrder: rest }]`
50
+ * - `last-fixed` `[{ anyOrder: init }, sLast]`
51
+ * - `first-and-last-fixed` `[s0, { anyOrder: middle }, sLast]`
52
+ * - `outside-in`, `random` `[{ anyOrder: all }]` (full any-order)
53
+ * - `middle-fixed` → set-membership completion (partial): the
54
+ * scrambled slots are the two ENDS (`u.shuffleOutside` — middle stays
55
+ * fixed), which is non-contiguous; neither the engine nor Mixpanel's
56
+ * any-order groups can express it.
44
57
  *
45
58
  * Returns a normalized result `{ completed, reached, ttcMs, mode, verificationKind }`.
46
- * `verificationKind` is `undefined` for sequential modes (full PASS/FAIL allowed),
47
- * `'partial'` for completion-only modes, `'informational'` for `random`.
59
+ * `verificationKind` is `undefined` for Mixpanel-comparable modes (full
60
+ * PASS/FAIL allowed), `'partial'` for `middle-fixed`'s completion-only check.
48
61
  *
49
62
  * @param {Array<Object>} userEvents
50
63
  * @param {string[]} steps
51
64
  * @param {Object} options
52
65
  * @param {string} [options.funnelOrder='sequential']
53
66
  * @param {number} [options.conversionWindowMs]
67
+ * @param {{unit: 'sessions', n: number}} [options.conversionWindow]
54
68
  * @param {boolean} [options.reentry]
55
69
  * @param {Array<Object>} [options.exclusionSteps]
56
70
  * @param {boolean | string[]} [options.trackStepProperties]
57
71
  * @param {boolean} [options.sessionScoped]
72
+ * @param {{fromMs?: number, toMs?: number}} [options.anchorRange] - step-0
73
+ * anchor bounds (P1.6.5) — honored by every order except `middle-fixed`
74
+ * (whose set-membership check has no anchor concept).
58
75
  */
59
76
  function evaluateFunnelByOrder(userEvents, steps, options = {}) {
60
77
  const order = options.funnelOrder || 'sequential';
61
78
  const sequentialOpts = {
62
79
  conversionWindowMs: options.conversionWindowMs,
80
+ conversionWindow: options.conversionWindow,
63
81
  reentry: options.reentry,
64
82
  exclusionSteps: options.exclusionSteps,
65
83
  trackStepProperties: options.trackStepProperties,
66
84
  sessionScoped: options.sessionScoped,
85
+ anchorRange: options.anchorRange,
67
86
  };
87
+ // Degenerate scrambled regions collapse to plain steps: an empty region
88
+ // means the order IS sequential, and a 1-element region shuffles to itself
89
+ // (the generator's shuffle of one element is the identity) — expressing it
90
+ // as a 1-member anyOrder block would change buffering semantics (chunk
91
+ // first-match vs anchor latest-match) for data that is in fact ordered.
92
+ const block = (arr) => arr.length === 1 ? arr[0] : { anyOrder: arr };
93
+ let engineSteps = /** @type {Array<*>} */ (steps);
68
94
  switch (order) {
69
- case 'sequential':
70
- case 'interrupt':
71
- case 'interrupted': {
72
- const r = /** @type {*} */ (evaluateFunnel(userEvents, steps, sequentialOpts));
73
- return { ...r, mode: order, verificationKind: undefined };
74
- }
75
- case 'first-fixed': {
76
- const stepZero = /** @type {*} */ (evaluateFunnel(userEvents, [steps[0]], { conversionWindowMs: options.conversionWindowMs }));
77
- if (!stepZero.completed) {
78
- return { completed: false, reached: -1, stepEvents: [], stepTimes: [], ttcMs: null, mode: 'first-fixed', verificationKind: 'partial' };
79
- }
80
- const rest = evaluateAnyOrderCompletion(userEvents, steps.slice(1));
81
- const completed = rest.completed;
82
- const reached = completed ? steps.length - 1 : 0;
83
- return {
84
- completed,
85
- reached,
86
- stepEvents: [],
87
- stepTimes: [],
88
- ttcMs: completed ? rest.completionTimeMs : null,
89
- mode: 'first-fixed',
90
- verificationKind: 'partial',
91
- };
92
- }
95
+ case 'first-fixed':
96
+ if (steps.length >= 2) engineSteps = [steps[0], block(steps.slice(1))];
97
+ break;
93
98
  case 'last-fixed':
94
- case 'middle-fixed':
99
+ if (steps.length >= 2) engineSteps = [block(steps.slice(0, -1)), steps[steps.length - 1]];
100
+ break;
95
101
  case 'first-and-last-fixed':
96
- case 'outside-in': {
102
+ if (steps.length >= 3) engineSteps = [steps[0], block(steps.slice(1, -1)), steps[steps.length - 1]];
103
+ break;
104
+ case 'outside-in':
105
+ case 'random':
106
+ // Full any-order group. `random` generation is a full shuffle;
107
+ // `outside-in` has no generator implementation (falls through to
108
+ // sequential order) — the any-order funnel accepts both.
109
+ if (steps.length >= 2) engineSteps = [{ anyOrder: steps.slice() }];
110
+ break;
111
+ case 'middle-fixed': {
97
112
  const r = evaluateAnyOrderCompletion(userEvents, steps);
98
113
  return {
99
114
  completed: r.completed,
@@ -105,28 +120,16 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
105
120
  verificationKind: 'partial',
106
121
  };
107
122
  }
108
- case 'random': {
109
- const r = evaluateAnyOrderCompletion(userEvents, steps);
110
- return {
111
- completed: r.completed,
112
- reached: r.completed ? steps.length - 1 : -1,
113
- stepEvents: [],
114
- stepTimes: [],
115
- ttcMs: r.completionTimeMs,
116
- mode: 'random',
117
- verificationKind: 'informational',
118
- };
119
- }
120
- default: {
121
- const r = /** @type {*} */ (evaluateFunnel(userEvents, steps, sequentialOpts));
122
- return { ...r, mode: order, verificationKind: undefined };
123
- }
123
+ default:
124
+ break; // sequential / interrupt / interrupted / unknown → as-is
124
125
  }
126
+ const r = /** @type {*} */ (evaluateFunnel(userEvents, engineSteps, sequentialOpts));
127
+ return { ...r, mode: order, verificationKind: undefined };
125
128
  }
126
129
 
127
130
  /**
128
131
  * @typedef {Object} EmulateOptions
129
- * @property {'frequencyByFrequency'|'funnelFrequency'|'aggregatePerUser'|'timeToConvert'|'attributedBy'|'sessionMetrics'|'retention'|'distinctCount'} type
132
+ * @property {'frequencyByFrequency'|'funnelFrequency'|'aggregatePerUser'|'timeToConvert'|'attributedBy'|'sessionMetrics'|'retention'|'distinctCount'|'eventBreakdown'|'uniques'|'lifecycle'|'topPaths'} type
130
133
  *
131
134
  * @property {string} [metricEvent]
132
135
  * @property {string} [breakdownByFrequencyOf]
@@ -141,7 +144,10 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
141
144
  * @property {'avg'|'sum'|'count'|'max'|'min'} [agg]
142
145
  *
143
146
  * @property {string} [fromEvent]
144
- * @property {string} [toEvent]
147
+ * @property {string} [toEvent] - timeToConvert pair sugar. v1.6.0 (P1.6.7):
148
+ * pass `steps` (>= 2 entries) instead for multi-step TTC; rows gain
149
+ * avg_ttc_s, step_counts, gap_avg_s, cumulative_avg_s
150
+ * (funnel_query.cpp:3355-3380 arithmetic — integer seconds, per-gap clamp).
145
151
  * @property {string} [breakdownByUserProperty]
146
152
  * @property {Array<Object>} [profiles]
147
153
  *
@@ -149,7 +155,61 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
149
155
  * @property {string} [attributionEvent]
150
156
  * @property {string} [attributionProperty]
151
157
  * @property {'firstTouch'|'lastTouch'} [model]
152
- * @property {number} [touchpointsLimit]
158
+ * @property {'first'|'all'} [perConversion] - attributedBy only: 'first' (default) evaluates
159
+ * only each user's first conversion (v1.5 behavior); 'all' runs attribution per conversion
160
+ * event (Mixpanel behavior — attributed_value_reader_read takes one
161
+ * event_time_ms per conversion).
162
+ *
163
+ * v1.6.0 eventBreakdown (Insights: total events segmented by a property):
164
+ * @property {string} [breakdownProperty]
165
+ * @property {number} [topN]
166
+ * @property {'sessions'|'unique'|'general'} [countType] - eventBreakdown +
167
+ * uniques (P1.7.3) accept 'sessions' only: count once per (user, session,
168
+ * segment) / distinct (user, session) pairs per bucket, deriving sessions
169
+ * via sessionize(). topPaths (P1.9) accepts all three (see extractFlows).
170
+ * @property {number} [sessionTimeoutMs]
171
+ *
172
+ * v1.6.0 (P1.7.2) sessionMetrics: `source: 'derived'` (default) re-derives
173
+ * sessions at query time via sessionize(); `'stamped'` keeps the v1.5
174
+ * stamped-session_id path. `sessionTimeoutMs`/`maxSessionMs` thread into
175
+ * sessionize. Every row reports `source` + `stampedDivergence`.
176
+ * @property {('count'|'duration'|'eventsPerSession')[]} [metrics]
177
+ * @property {'derived'|'stamped'} [source]
178
+ * @property {number} [maxSessionMs]
179
+ *
180
+ * v1.6.0 uniques (Insights: unique users per interval / XAU / cumulative):
181
+ * @property {Object<string, *>} [where]
182
+ * @property {('day'|'week'|'month'|'range')} [unit]
183
+ * @property {number} [rollingWindow]
184
+ * @property {boolean} [cumulative]
185
+ * @property {boolean} [firstTimeOnly]
186
+ *
187
+ * v1.6.0 lifecycle (P1.8 — the Lifecycle Cohort Analysis classification;
188
+ * New / Retained / Resurrected / Dormant user counts per tiled period):
189
+ * @property {string} [valueMomentEvent] - The event that defines "active".
190
+ * @property {7|30} [periodDays] - The two LCA template variants.
191
+ *
192
+ * v1.6.0 topPaths (P1.9 — the Flows report; anchored event paths aggregated
193
+ * into a pruned prefix tree; see flows.js for the ARB citations):
194
+ * @property {Array<string|{event: string, where?: Object<string, *>}>} [anchors]
195
+ * @property {number|number[]} [forward] - Steps kept AFTER each anchor
196
+ * (linear buffer, keeps the FIRST N). Scalar broadcasts; default 4 mirrors
197
+ * the UI's expansion default (spec contract, not an ARB constant).
198
+ * @property {number|number[]} [reverse] - Steps kept BEFORE each anchor
199
+ * (ring buffer, keeps the LAST N). Scalar broadcasts; default 0.
200
+ * @property {'list'|'sankey'} [output] - Result shape (default 'list').
201
+ * @property {number} [cardinalityThreshold] - Per-level top-N before
202
+ * coalescing into $mp_uncommon_flows_events (default 50 list / 3 sankey —
203
+ * bookmark.py:96/:110). In list mode the same value caps the returned
204
+ * rows (flows_merger.cpp:406-410; 0 = no row cap).
205
+ * @property {'end'|'start'} [listSortPriority] - List row ordering, by leaf
206
+ * count ('end', default) or slot-0 count ('start') — bookmark.py:95.
207
+ * @property {string[]} [hiddenEvents] - Non-anchor steps to drop (anchor
208
+ * names exempt — query/flows.go filterAnchorEventNames).
209
+ * @property {string[]} [visibleEvents] - Non-anchor allow-list; hidden wins
210
+ * (query/flows.go filteredVisibleEventSelectors).
211
+ * @property {boolean} [collapseRepeated] - Suppress consecutive repeats
212
+ * (default false — flows_params_utils.py:37).
153
213
  *
154
214
  * v1.5.0 funnel extensions (apply to funnelFrequency + timeToConvert sequential modes):
155
215
  * @property {boolean} [reentry]
@@ -157,6 +217,23 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
157
217
  * @property {boolean | string[]} [trackStepProperties]
158
218
  * @property {boolean} [sessionScoped]
159
219
  *
220
+ * v1.6.0 funnel count/window extensions (funnelFrequency, sequential order modes
221
+ * only — findings #15). `countMode` follows evaluateFunnel: 'uniques' (default,
222
+ * one result per user), 'totals' (every attempt counts a conversion row),
223
+ * 'sessions' (Mixpanel's session count type — rewritten to totals +
224
+ * 1-session window per api/version_2_0/arb_funnels/validate.py
225
+ * __validate_sessions). `holdPropertyConstant` splits each user into parallel
226
+ * per-property-value sub-funnels (evaluateFunnelHPC — scalar values only).
227
+ * `conversionWindow` is the session-count window ({ unit: 'sessions', n ≤ 12 });
228
+ * mutually exclusive with `conversionWindowMs`.
229
+ * @property {'uniques'|'totals'|'sessions'} [countMode]
230
+ * @property {string} [holdPropertyConstant]
231
+ * @property {{unit: 'sessions', n: number}} [conversionWindow]
232
+ * @property {string} [funnelOrder] - Funnel step-order mode (see
233
+ * evaluateFunnelByOrder); defaults to 'sequential'.
234
+ * @property {{from?: number|string, to?: number|string}} [cohortWindow] -
235
+ * Retention birth window (inclusive bounds — see retention()).
236
+ *
160
237
  * @property {Map<string,string>} [identityMap]
161
238
  *
162
239
  * Cross-cutting time-bucketed output:
@@ -170,6 +247,21 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
170
247
  * numerical aggregation. Mixpanel `normal_query.cpp:352-356` emits zero
171
248
  * rows for empty intervals; we use the `_empty` marker instead of
172
249
  * guessing a per-type zero-row template.
250
+ *
251
+ * v1.6.0 (P1.6.5) — step-0-anchored trends: under `timeBucket`, the types
252
+ * `funnelFrequency` and `timeToConvert` and `retention` follow Mixpanel's
253
+ * interval rule "step 0 in [start, stop); steps 1+ in
254
+ * [start, stop + conversion window)" (funnel_query.cpp:1398-1401;
255
+ * history.cpp:437-440 for retention returns): a conversion or return
256
+ * spilling past the bucket edge is credited to the bucket that ANCHORED it.
257
+ * P1.6.6 extended this to every funnel order that routes through the engine's
258
+ * anyOrder blocks (all orders except `middle-fixed`, whose set-membership
259
+ * check keeps plain event partitioning).
260
+ * @property {{fromMs?: number, toMs?: number}} [anchorRange] - Step-0 anchor
261
+ * bounds for `funnelFrequency`/`timeToConvert` (set automatically by the
262
+ * timeBucket wrapper; may be passed directly for a single-interval run).
263
+ * Events outside `[fromMs, toMs)` cannot anchor step 0 or count on the
264
+ * frequency axis, but can complete later steps.
173
265
  */
174
266
 
175
267
  /**
@@ -178,7 +270,8 @@ function evaluateFunnelByOrder(userEvents, steps, options = {}) {
178
270
  *
179
271
  * @param {Array<Object>} events
180
272
  * @param {EmulateOptions} config
181
- * @returns {Array<Object>} Breakdown table rows.
273
+ * @returns {Array<Object>|Object} Breakdown table rows (every type except
274
+ * 'topPaths', which returns aggregateFlows' single result object).
182
275
  */
183
276
  export function emulateBreakdown(events, config) {
184
277
  if (!Array.isArray(events)) throw new Error('emulateBreakdown: events must be an array');
@@ -207,13 +300,113 @@ export function emulateBreakdown(events, config) {
207
300
  // emits zero rows for empty intervals). Consumers MUST filter `r._empty`
208
301
  // before any aggregation.
209
302
  if (config.timeBucket) {
303
+ if (config.type === 'uniques') {
304
+ // `uniques` produces its own time axis (unit/rollingWindow/cumulative).
305
+ // Pre-partitioning here would break rolling look-back windows and the
306
+ // cumulative running set, which both span bucket boundaries.
307
+ throw new Error('emulateBreakdown: type "uniques" does not compose with timeBucket — use `unit` instead');
308
+ }
309
+ if (config.type === 'lifecycle') {
310
+ // Classification is cross-period by construction (retained/resurrected/
311
+ // dormant all read T−1 and earlier) — pre-partitioning would blind it.
312
+ throw new Error('emulateBreakdown: type "lifecycle" does not compose with timeBucket — it tiles its own period axis');
313
+ }
314
+ if (config.type === 'topPaths') {
315
+ // Flows aggregates ONE path universe over the whole query range (a
316
+ // single prefix tree — flows_query.cpp end-of-data flush); there is
317
+ // no per-interval trend variant to partition into.
318
+ throw new Error('emulateBreakdown: type "topPaths" does not compose with timeBucket — flows aggregate one path universe over the range');
319
+ }
210
320
  const range = config.timeBucketRange || {};
211
321
  const buckets = partitionByTimeBucket(events, config.timeBucket, range);
212
322
  // Pass the pre-built identityMap into recursive calls so the auto-build
213
323
  // branch above is a no-op per bucket (would otherwise rebuild N times).
214
324
  const inner = { ...config, timeBucket: undefined, timeBucketRange: undefined, identityMap };
215
325
  const out = [];
216
- for (const { period, events: evs } of buckets) {
326
+
327
+ // v1.6.0 (P1.6.5): step-0-anchored trend types. Mixpanel evaluates each
328
+ // trend interval as "step 0 in [start, stop); steps 1+ in
329
+ // [start, stop + conversion window)" (funnel_query.cpp:1398-1401), and
330
+ // retention's birth-in-interval with returns spilling past the interval
331
+ // edge (history.cpp:437-440). Plain event-partitioning would truncate
332
+ // any window spanning a bucket edge, so:
333
+ // - retention: run each bucket over the FULL stream, intersecting
334
+ // `cohortWindow` with the bucket bounds (births anchor in-bucket,
335
+ // returns spill freely). ARB keeps a separate retention history
336
+ // per (user, interval) — a user re-births in EVERY interval that
337
+ // contains a cohort-side match (retention_query.cpp:1334-1341), so
338
+ // the same user can appear in multiple buckets' cohorts;
339
+ // - funnelFrequency / timeToConvert: slice to [start, stop + window)
340
+ // and pass `anchorRange` down so step 0 only anchors in
341
+ // [start, stop).
342
+ // v1.6.0 (P1.6.6): every funnel order except 'middle-fixed' now routes
343
+ // through the engine (anyOrder blocks honor anchorRange — the funnel's
344
+ // FIRST POSITION, whatever slot fills it, must anchor in [start, stop)).
345
+ // 'middle-fixed' keeps plain partitioning: its scrambled slots are the
346
+ // two ends (non-contiguous), so it stays on the set-membership helper,
347
+ // which has no anchor concept.
348
+ const anchorableOrder = config.funnelOrder !== 'middle-fixed';
349
+ const anchored = config.type === 'retention'
350
+ || ((config.type === 'funnelFrequency' || config.type === 'timeToConvert') && anchorableOrder);
351
+
352
+ if (anchored) {
353
+ for (const { period } of buckets) {
354
+ const { startMs, endMs } = bucketBoundsMs(period, config.timeBucket);
355
+ let rows;
356
+ if (config.type === 'retention') {
357
+ const cw = config.cohortWindow || {};
358
+ const from = Math.max(startMs, cw.from != null ? toMs(cw.from) : -Infinity);
359
+ const to = Math.min(endMs - 1, cw.to != null ? toMs(cw.to) : Infinity); // cohortWindow bounds are inclusive
360
+ rows = from > to ? [] : emulateBreakdown(events, { ...inner, cohortWindow: { from, to } });
361
+ } else {
362
+ const spillMs = typeof config.conversionWindowMs === 'number'
363
+ ? config.conversionWindowMs
364
+ : (config.conversionWindow && typeof config.conversionWindow.n === 'number'
365
+ // session windows: ARB's per-step check is ordinal-only, but
366
+ // histories terminate conversion_window_max_length_seconds
367
+ // (n × SECONDS_PER_DAY for sessions, unit.c:14) past the
368
+ // INTERVAL END (funnel_query.cpp:1620) — this slice mirrors that.
369
+ ? config.conversionWindow.n * 86400_000
370
+ : Infinity);
371
+ const sliceEnd = endMs + spillMs;
372
+ const slice = events.filter(e => {
373
+ const t = toMs(e && e.time);
374
+ return Number.isFinite(t) && t >= startMs && t < sliceEnd;
375
+ });
376
+ rows = emulateBreakdown(slice, { ...inner, anchorRange: { fromMs: startMs, toMs: endMs } });
377
+ }
378
+ if (rows.length) {
379
+ for (const r of rows) out.push({ period, ...r });
380
+ } else {
381
+ out.push({ period, _empty: true });
382
+ }
383
+ }
384
+ return out;
385
+ }
386
+
387
+ // v1.6.0 fix round (B5): firstTimeOnly must see the FULL stream.
388
+ // Mixpanel computes ONE first_event_time per user over the whole
389
+ // lookback (~5y — event_selector.py:125-149). Recursing with the raw
390
+ // per-bucket slice would re-elect a "first" inside every bucket,
391
+ // counting each active user once per bucket instead of exactly once
392
+ // overall. Filter the FULL stream, repartition, clear the flag for
393
+ // the inner calls — the survivor lands in its one true bucket.
394
+ //
395
+ // countType 'sessions' needs NO such hoist: sessions never cross UTC
396
+ // midnight (unconditional daySplit, sessionize.js buildUserSessions —
397
+ // ARB resets session state at the day boundary) and every timeBucket
398
+ // unit (day/week/month) cuts at midnights, so each session's events
399
+ // are wholly inside one bucket and slice re-derivation reproduces the
400
+ // full-stream boundaries exactly (locked by the B5 invariant tests in
401
+ // event-breakdown.test.js).
402
+ let genericBuckets = buckets;
403
+ if (config.type === 'eventBreakdown' && config.firstTimeOnly) {
404
+ const firsts = filterFirstTimeEver(events, { event: config.event, identityMap });
405
+ genericBuckets = partitionByTimeBucket(firsts, config.timeBucket, range);
406
+ inner.firstTimeOnly = false;
407
+ }
408
+
409
+ for (const { period, events: evs } of genericBuckets) {
217
410
  const rows = emulateBreakdown(evs, inner);
218
411
  if (rows.length) {
219
412
  for (const r of rows) out.push({ period, ...r });
@@ -234,6 +427,10 @@ export function emulateBreakdown(events, config) {
234
427
  case 'sessionMetrics': return sessionMetrics(events, /** @type {*} */ (cfg));
235
428
  case 'retention': return retention(events, /** @type {*} */ (cfg));
236
429
  case 'distinctCount': return distinctCount(events, /** @type {*} */ (cfg));
430
+ case 'eventBreakdown': return eventBreakdown(events, /** @type {*} */ (cfg));
431
+ case 'uniques': return uniques(events, /** @type {*} */ (cfg));
432
+ case 'lifecycle': return lifecycle(events, /** @type {*} */ (cfg));
433
+ case 'topPaths': return topPaths(events, /** @type {*} */ (cfg));
237
434
  default: throw new Error(`emulateBreakdown: unknown type "${config.type}"`);
238
435
  }
239
436
  }
@@ -267,25 +464,130 @@ function frequencyByFrequency(events, { metricEvent, breakdownByFrequencyOf, per
267
464
  // Step progression uses the greedy single-pass funnel engine
268
465
  // (funnel-engine.js → history.cpp). Cohort breakdown axis uses distinct-period
269
466
  // counting (addiction_query.cpp).
467
+ //
468
+ // v1.6.0 (P1.6.4): rows carry an `excluded` column — histories terminated by
469
+ // an exclusion step tally at step_index = the boundary they failed to cross
470
+ // (ARB `fr->excluded[reached + 1]`, funnel_query.cpp:3422; uniques variant
471
+ // `fr->excluded_uniques[reached + 1]`, :3337). A boundary with exclusions but
472
+ // zero conversions still emits a row.
270
473
 
271
- function funnelFrequency(events, { steps, breakdownByFrequencyOf, conversionWindowMs, periodUnit = 'day', funnelOrder = 'sequential', identityMap, reentry, exclusionSteps, trackStepProperties, sessionScoped }) {
474
+ function funnelFrequency(events, { steps, breakdownByFrequencyOf, conversionWindowMs, conversionWindow, periodUnit = 'day', funnelOrder = 'sequential', identityMap, reentry, exclusionSteps, trackStepProperties, sessionScoped, countMode, holdPropertyConstant, anchorRange }) {
272
475
  if (!Array.isArray(steps) || !steps.length) throw new Error('funnelFrequency requires steps[]');
273
476
  if (!breakdownByFrequencyOf) throw new Error('funnelFrequency requires breakdownByFrequencyOf');
477
+
478
+ // v1.6.0 (findings #15): countMode 'totals'/'sessions' and holdPropertyConstant
479
+ // route to the greedy single-pass primitives directly — attempt histories and
480
+ // HPC sub-funnels have no analog in the any-order completion modes.
481
+ const isSequentialOrder = funnelOrder === 'sequential' || funnelOrder === 'interrupt' || funnelOrder === 'interrupted';
482
+ const isTotalsMode = countMode === 'totals' || countMode === 'sessions';
483
+ if ((isTotalsMode || holdPropertyConstant) && !isSequentialOrder) {
484
+ throw new Error(`funnelFrequency: countMode '${countMode}' / holdPropertyConstant require a sequential funnel order (got '${funnelOrder}')`);
485
+ }
486
+ // HPC buckets events by property value BEFORE evaluation (evaluateFunnelHPC),
487
+ // but Mixpanel derives session boundaries from the user's FULL event stream —
488
+ // sessionizing each bucket independently would merge across the gaps left by
489
+ // removed events and produce wrong ordinals. Refuse rather than mis-count.
490
+ if (holdPropertyConstant && (countMode === 'sessions' || conversionWindow)) {
491
+ throw new Error('funnelFrequency: holdPropertyConstant cannot combine with session-count conversion windows — session boundaries derive from the full event stream, but HPC evaluates per-property-value event subsets');
492
+ }
493
+
274
494
  const userEvents = groupByUser(events, identityMap);
275
495
  const result = [];
276
496
  const conversions = new Map(); // `${stepIdx}|${b}` → count
497
+ const excludedCounts = new Map(); // `${stepIdx}|${b}` → count (stepIdx = excludedAtStep)
498
+ const funnelOpts = { conversionWindowMs, conversionWindow, countMode, reentry, exclusionSteps, trackStepProperties, sessionScoped, anchorRange };
277
499
  for (const [, evs] of userEvents) {
278
- // v1.5: dispatch on funnel.order so non-sequential modes don't return 0% trivially.
279
- const r = evaluateFunnelByOrder(evs, steps, { conversionWindowMs, funnelOrder, reentry, exclusionSteps, trackStepProperties, sessionScoped });
280
- const b = countDistinctPeriods(evs, breakdownByFrequencyOf, /** @type {*} */ (periodUnit));
281
- for (let s = 0; s <= r.reached; s++) {
500
+ // Frequency axis: under a trend interval (anchorRange set by the
501
+ // timeBucket wrapper) count only in-interval events spill events past
502
+ // the bucket edge exist solely for later-step matching, not the cohort
503
+ // axis (which would otherwise shift users between frequency cohorts
504
+ // depending on the conversion window length).
505
+ const axisEvs = anchorRange
506
+ ? evs.filter(e => {
507
+ const t = toMs(e && e.time);
508
+ return Number.isFinite(t)
509
+ && (anchorRange.fromMs == null || t >= anchorRange.fromMs)
510
+ && (anchorRange.toMs == null || t < anchorRange.toMs);
511
+ })
512
+ : evs;
513
+ const b = countDistinctPeriods(axisEvs, breakdownByFrequencyOf, /** @type {*} */ (periodUnit));
514
+ // One entry per credited history: uniques = the user's single (or best-
515
+ // across-HPC-values) progression; totals/sessions = every attempt.
516
+ /** @type {number[]} */
517
+ const reachedList = [];
518
+ // v1.6.0 (P1.6.4): exclusion boundary indexes (= reached + 1 of a
519
+ // terminated history) — ARB counts them per step slot:
520
+ // `fr->excluded[reached + 1]` (funnel_query.cpp:3422, totals) /
521
+ // `fr->excluded_uniques[reached + 1]` (funnel_query.cpp:3337, uniques).
522
+ /** @type {number[]} */
523
+ const excludedList = [];
524
+ if (holdPropertyConstant) {
525
+ const perValue = evaluateFunnelHPC(evs, steps, holdPropertyConstant, funnelOpts);
526
+ if (isTotalsMode) {
527
+ // Every attempt in every value bucket counts (totals semantics).
528
+ for (const r of perValue.values()) {
529
+ for (const a of /** @type {Array<*>} */ (r)) {
530
+ reachedList.push(a.reached);
531
+ if (a.terminatedByExclusion) excludedList.push(a.excludedAtStep);
532
+ }
533
+ }
534
+ } else {
535
+ // Uniques: per-step distinct-user counting over parallel histories.
536
+ // A user appears at step s when ANY value thread reached ≥ s, and
537
+ // progression is contiguous from 0 — so the union of credited steps
538
+ // is 0..max(reached). Credit the user ONCE at the furthest thread.
539
+ let best = -1;
540
+ let bestExcluded = false;
541
+ for (const r of perValue.values()) {
542
+ const reached = /** @type {*} */ (r).reached;
543
+ if (reached > best) {
544
+ best = reached;
545
+ bestExcluded = !!(/** @type {*} */ (r).terminatedByExclusion);
546
+ } else if (reached === best && /** @type {*} */ (r).terminatedByExclusion) {
547
+ // ARB-UNCERTAIN: HPC exclusion uniques. Conservative reading —
548
+ // the user counts as excluded when ANY furthest-reaching value
549
+ // thread was terminated by exclusion (each per-value history
550
+ // aggregates independently in ARB; per-user dedup across
551
+ // threads at the same boundary is our distinct-user layer).
552
+ bestExcluded = true;
553
+ }
554
+ }
555
+ if (best >= 0) {
556
+ reachedList.push(best);
557
+ if (bestExcluded) excludedList.push(best + 1);
558
+ }
559
+ }
560
+ } else if (isTotalsMode) {
561
+ const attempts = /** @type {Array<*>} */ (evaluateFunnel(evs, steps, funnelOpts));
562
+ for (const a of attempts) {
563
+ reachedList.push(a.reached);
564
+ if (a.terminatedByExclusion) excludedList.push(a.excludedAtStep);
565
+ }
566
+ } else {
567
+ // v1.5: dispatch on funnel.order so non-sequential modes don't return 0% trivially.
568
+ const r = evaluateFunnelByOrder(evs, steps, { conversionWindowMs, conversionWindow, funnelOrder, reentry, exclusionSteps, trackStepProperties, sessionScoped, anchorRange });
569
+ reachedList.push(r.reached);
570
+ if (/** @type {*} */ (r).terminatedByExclusion) excludedList.push(/** @type {*} */ (r).excludedAtStep);
571
+ }
572
+ for (const reached of reachedList) {
573
+ for (let s = 0; s <= reached; s++) {
574
+ const key = `${s}|${b}`;
575
+ conversions.set(key, (conversions.get(key) || 0) + 1);
576
+ }
577
+ }
578
+ for (const s of excludedList) {
282
579
  const key = `${s}|${b}`;
283
- conversions.set(key, (conversions.get(key) || 0) + 1);
580
+ excludedCounts.set(key, (excludedCounts.get(key) || 0) + 1);
284
581
  }
285
582
  }
286
- for (const [key, count] of conversions) {
583
+ const rowKeys = new Set([...conversions.keys(), ...excludedCounts.keys()]);
584
+ for (const key of rowKeys) {
287
585
  const [s, b] = key.split('|').map(Number);
288
- result.push({ step: steps[s], step_index: s, breakdown_freq: b, conversions: count, conversion_pct: 0 });
586
+ result.push({
587
+ step: steps[s], step_index: s, breakdown_freq: b,
588
+ conversions: conversions.get(key) || 0, conversion_pct: 0,
589
+ excluded: excludedCounts.get(key) || 0,
590
+ });
289
591
  }
290
592
  // Conversion % at each step relative to its own breakdown_freq's step-0 baseline.
291
593
  const baseline = new Map();
@@ -359,14 +661,38 @@ function aggregatePerUser(events, { event, property, agg = 'avg', breakdownByFre
359
661
 
360
662
  // ── Time to Convert (Funnel TTC, broken down by user property) ──
361
663
  //
362
- // Step pair matched via the greedy funnel engine (history.cpp). When the
363
- // funnel completes via the engine, ttcMs = stepTimes[1] - stepTimes[0].
364
- // Differs from the old "first occurrence of fromEvent then first occurrence
365
- // of toEvent after it" logic by enforcing the same temporal rules Mixpanel
366
- // uses for funnel matching.
664
+ // Steps matched via the greedy funnel engine (history.cpp). v1.6.0 (P1.6.7)
665
+ // aligns the arithmetic with ARB's funnel aggregation
666
+ // (funnel_query.cpp:3355-3380) and extends past 2-step pairs: pass `steps`
667
+ // (>= 2 entries, funnelOrder/exclusions apply) or keep the
668
+ // `fromEvent`/`toEvent` pair sugar.
669
+ //
670
+ // Output per segment row (rows are emitted for segments with >= 1 full
671
+ // conversion — the TTC chart is a converted-users report):
672
+ // - avg_ttc_ms / median_ttc_ms — over converted users; ttcMs is `$ttc` at
673
+ // ms resolution (MAX over slot times − position-0 time,
674
+ // history.cpp:914-918).
675
+ // - avg_ttc_s — mean of the integer-second `$ttc` values (floor'd per user
676
+ // BEFORE averaging, matching the computed property's resolution).
677
+ // - step_counts[i] — attempts that reached position i, drop-offs included
678
+ // (fr->counts, funnel_query.cpp:3362 — the loop runs for every history).
679
+ // - gap_avg_s[g] / cumulative_avg_s[g] — sum_deltas[i]/counts[i] and
680
+ // sum_deltas_from_start[i]/counts[i] for position i = g+1
681
+ // (funnel_query.cpp:3374-3380): per-gap deltas are integer seconds
682
+ // clamped to 0 per gap; the cumulative track sums the CLAMPED deltas.
683
+ // `null` when no attempt reached that position.
367
684
 
368
- function timeToConvert(events, { fromEvent, toEvent, breakdownByUserProperty, profiles = [], funnelOrder = 'sequential', conversionWindowMs, identityMap, reentry, exclusionSteps, sessionScoped }) {
369
- if (!fromEvent || !toEvent) throw new Error('timeToConvert requires fromEvent and toEvent');
685
+ function timeToConvert(events, { fromEvent, toEvent, steps: stepsOpt, breakdownByUserProperty, profiles = [], funnelOrder = 'sequential', conversionWindowMs, identityMap, reentry, exclusionSteps, sessionScoped, anchorRange }) {
686
+ let steps;
687
+ if (Array.isArray(stepsOpt) && stepsOpt.length) {
688
+ if (fromEvent || toEvent) throw new Error('timeToConvert takes either steps[] or fromEvent/toEvent, not both');
689
+ if (stepsOpt.length < 2) throw new Error('timeToConvert steps[] needs at least 2 steps');
690
+ steps = stepsOpt;
691
+ } else {
692
+ if (!fromEvent || !toEvent) throw new Error('timeToConvert requires fromEvent and toEvent (or steps[])');
693
+ steps = [fromEvent, toEvent];
694
+ }
695
+ const numSteps = steps.length;
370
696
  const userEvents = groupByUser(events, identityMap);
371
697
  const profileByUid = new Map();
372
698
  for (const p of profiles) {
@@ -374,62 +700,117 @@ function timeToConvert(events, { fromEvent, toEvent, breakdownByUserProperty, pr
374
700
  const uid = p.distinct_id || p.user_id;
375
701
  if (uid) profileByUid.set(uid, p);
376
702
  }
377
- const buckets = new Map(); // segValue [ttcMs]
703
+ // segValue { ttcs, ttcSecs, counts, sumGap, sumCum }
704
+ const buckets = new Map();
705
+ const acc = (segValue) => {
706
+ if (!buckets.has(segValue)) buckets.set(segValue, {
707
+ ttcs: [], ttcSecs: [],
708
+ counts: new Array(numSteps).fill(0),
709
+ sumGap: new Array(numSteps - 1).fill(0),
710
+ sumCum: new Array(numSteps - 1).fill(0),
711
+ });
712
+ return buckets.get(segValue);
713
+ };
378
714
  for (const [uid, evs] of userEvents) {
379
- // v1.5: respect funnel.order. For random mode, ttcMs is informational
380
- // (lastSeenTime - firstSeenTime), not Mixpanel TTC.
381
- const r = evaluateFunnelByOrder(evs, [fromEvent, toEvent], { funnelOrder, conversionWindowMs, reentry, exclusionSteps, sessionScoped });
382
- if (!r.completed || r.ttcMs === null || !Number.isFinite(r.ttcMs) || r.ttcMs < 0) continue;
715
+ // v1.5: respect funnel.order. middle-fixed stays set-membership
716
+ // (verificationKind 'partial') no gap arrays there.
717
+ const r = evaluateFunnelByOrder(evs, steps, { funnelOrder, conversionWindowMs, reentry, exclusionSteps, sessionScoped, anchorRange });
383
718
  const profile = profileByUid.get(uid);
384
719
  const segValue = breakdownByUserProperty
385
720
  ? (profile ? (profile[breakdownByUserProperty] ?? 'unknown') : 'unknown')
386
721
  : 'all';
387
- if (!buckets.has(segValue)) buckets.set(segValue, []);
388
- buckets.get(segValue).push(r.ttcMs);
389
- }
390
- return [...buckets.entries()].map(([seg, ttcs]) => ({
391
- segment_value: seg,
392
- user_count: ttcs.length,
393
- avg_ttc_ms: ttcs.reduce((a, x) => a + x, 0) / ttcs.length,
394
- median_ttc_ms: median(ttcs),
395
- })).sort((x, y) => String(x.segment_value).localeCompare(String(y.segment_value)));
722
+ const a = acc(segValue);
723
+ // funnel_query.cpp:3359-3380 — EVERY attempt (converted or not)
724
+ // records counts[i] for i <= reached and gap sums for 1 <= i <= reached.
725
+ if (typeof r.reached === 'number' && r.reached >= 0 && Array.isArray(r.gapSeconds)) {
726
+ for (let p = 0; p <= r.reached && p < numSteps; p++) a.counts[p]++;
727
+ for (let g = 0; g < r.gapSeconds.length && g < numSteps - 1; g++) {
728
+ a.sumGap[g] += r.gapSeconds[g];
729
+ a.sumCum[g] += r.gapSecondsFromStart[g];
730
+ }
731
+ }
732
+ if (!r.completed || r.ttcMs === null || !Number.isFinite(r.ttcMs) || r.ttcMs < 0) continue;
733
+ a.ttcs.push(r.ttcMs);
734
+ // $ttc is integer seconds (history.cpp:917-918); middle-fixed partial
735
+ // results carry no ttcSeconds — floor their informational ttcMs.
736
+ a.ttcSecs.push(typeof r.ttcSeconds === 'number' ? r.ttcSeconds : Math.floor(r.ttcMs / 1000));
737
+ }
738
+ return [...buckets.entries()]
739
+ .filter(([, a]) => a.ttcs.length > 0)
740
+ .map(([seg, a]) => ({
741
+ segment_value: seg,
742
+ user_count: a.ttcs.length,
743
+ avg_ttc_ms: a.ttcs.reduce((x, y) => x + y, 0) / a.ttcs.length,
744
+ median_ttc_ms: median(a.ttcs),
745
+ avg_ttc_s: a.ttcSecs.reduce((x, y) => x + y, 0) / a.ttcSecs.length,
746
+ step_counts: a.counts,
747
+ gap_avg_s: a.sumGap.map((s, g) => a.counts[g + 1] > 0 ? s / a.counts[g + 1] : null),
748
+ cumulative_avg_s: a.sumCum.map((s, g) => a.counts[g + 1] > 0 ? s / a.counts[g + 1] : null),
749
+ }))
750
+ .sort((x, y) => String(x.segment_value).localeCompare(String(y.segment_value)));
396
751
  }
397
752
 
398
753
  // ── Attributed By (first-/last-touch attribution by event property value) ──
399
754
  //
400
- // Touchpoint cap: max 10 touchpoints in lookback window
401
- // (`TOUCHPOINTS_LIMIT = 10` in attributed_value_reader.cpp). For first/last
402
- // touch the cap matters when the user has > 10 touches before conversion;
403
- // the cap shifts which touches enter the candidate pool.
755
+ // NO touchpoint cap for FIRST/LAST models. `TOUCHPOINTS_LIMIT = 10`
756
+ // (backend/libquery/properties_over_time/attributed_value_reader.cpp:16)
757
+ // rides in whoval_reader_params on every read, but the FIRST/LAST paths
758
+ // execute hard-`LIMIT 1` statements (whoval/read.cpp:173-192 first_stmt_,
759
+ // :643-655 get_first/last_value) — only `sorted_list_stmt_` (`LIMIT ?4`,
760
+ // read.cpp:595, get_list_values :681-685) consumes the cap, and that
761
+ // statement serves multi-touch list models (LINEAR/PARTICIPATION/
762
+ // TIME_DECAY) this emulator doesn't implement. FIRST = globally first
763
+ // touch in the lookback window, however many touches precede it.
764
+ //
765
+ // Per-conversion semantics: Mixpanel runs attribution once PER conversion
766
+ // event — `attributed_value_reader_read` takes a single `event_time_ms` and
767
+ // builds a fresh lookback read ending at that conversion.
768
+ // `perConversion: 'all'` matches that; the default `'first'` keeps the v1.5
769
+ // one-conversion-per-user behavior for back-compat.
770
+ //
771
+ // ⚠ TOUCHPOINT SEAM (findings #2, closed-as-documented): the GENERATOR
772
+ // samples which events get UTM stamps uniformly across a user's lifetime
773
+ // (maxTouchpointsPerUser sampling in lib/orchestrators/user-loop.js —
774
+ // `sampled across lifetime`), while this verifier and Mixpanel both read
775
+ // touchpoints BEFORE each conversion. A user with more eligible events
776
+ // than the generator cap can have touches that never got stamped — and
777
+ // stamped touches can postdate every conversion. Divergence is
778
+ // theoretical below ~10 eligible events per user (the generator cap);
779
+ // attribution-engineering hooks should OVERWRITE engine-stamped UTMs near
780
+ // the conversion rather than relying on the lifetime-uniform sampling
781
+ // (hook rule 10, CLAUDE.md).
404
782
 
405
783
  function attributedBy(events, {
406
784
  conversionEvent,
407
785
  attributionEvent,
408
786
  attributionProperty,
409
787
  model = 'firstTouch',
410
- touchpointsLimit = 10,
788
+ perConversion = 'first',
411
789
  identityMap,
412
790
  }) {
413
791
  if (!conversionEvent || !attributionEvent || !attributionProperty) {
414
792
  throw new Error('attributedBy requires conversionEvent, attributionEvent, attributionProperty');
415
793
  }
794
+ if (perConversion !== 'first' && perConversion !== 'all') {
795
+ throw new Error(`attributedBy: unknown perConversion "${perConversion}" — use 'first' or 'all'`);
796
+ }
416
797
  const userEvents = groupByUser(events, identityMap);
417
798
  const counts = new Map();
418
799
  for (const [, evs] of userEvents) {
419
800
  const sorted = sortByTime(evs);
420
- const conversion = sorted.find(e => e && e.event === conversionEvent);
421
- if (!conversion) continue;
422
- const conversionTime = toMs(conversion.time);
423
- const allTouches = sorted.filter(e =>
424
- e && e.event === attributionEvent && toMs(e.time) <= conversionTime
425
- );
426
- if (!allTouches.length) continue;
427
- // Cap to the last `touchpointsLimit` touches in the lookback window.
428
- // (When touch count <= cap, this is a no-op.)
429
- const touches = allTouches.slice(-touchpointsLimit);
430
- const touch = model === 'lastTouch' ? touches[touches.length - 1] : touches[0];
431
- const v = touch[attributionProperty] ?? 'unknown';
432
- counts.set(v, (counts.get(v) || 0) + 1);
801
+ const conversions = perConversion === 'all'
802
+ ? sorted.filter(e => e && e.event === conversionEvent)
803
+ : sorted.filter(e => e && e.event === conversionEvent).slice(0, 1);
804
+ for (const conversion of conversions) {
805
+ const conversionTime = toMs(conversion.time);
806
+ const allTouches = sorted.filter(e =>
807
+ e && e.event === attributionEvent && toMs(e.time) <= conversionTime
808
+ );
809
+ if (!allTouches.length) continue;
810
+ const touch = model === 'lastTouch' ? allTouches[allTouches.length - 1] : allTouches[0];
811
+ const v = touch[attributionProperty] ?? 'unknown';
812
+ counts.set(v, (counts.get(v) || 0) + 1);
813
+ }
433
814
  }
434
815
  return [...counts.entries()].map(([source, count]) => ({
435
816
  attribution_value: source,
@@ -437,85 +818,298 @@ function attributedBy(events, {
437
818
  })).sort((a, b) => b.conversions - a.conversions);
438
819
  }
439
820
 
440
- // ── Retention (birth-anchored day buckets) ───────────────────────────────────
821
+ // ── Retention (birth-anchored buckets) ───────────────────────────────────────
441
822
  //
442
823
  // Reference: backend/arb/reader/queries/retention_query.cpp
443
824
  //
444
- // Bucketing rule (retention_query.cpp:1227-1231):
445
- // time_to_retention_event_s = retention_event_time_s - first_event_time_s
825
+ // Bucketing rule (retention_query.cpp:1258-1262):
826
+ // time_to_retention_event_s = retention_event_time_s - aligned_birth_time_s
446
827
  // bucket = floor(time_to_retention_event_s / bucket_seconds)
828
+ // Bucket seconds per unit (libquery/time/unit.c:5-16 + libquery/util.h:265-273):
829
+ // hour = 3600s, day = 86400s, week = 7d, month = 31d FIXED ("maximum
830
+ // possible seconds in a month" — NOT calendar months).
831
+ //
832
+ // Birth-vs-return gate (retention_query.cpp:1120-1139,
833
+ // retention_query_event_occurs_after_birth — cites COR-233):
834
+ // `<=` applies ONLY when birthCanRetain AND the return event ALSO matches
835
+ // the birth filter (`matches_first`). Distinct birth/return events at the
836
+ // same ms stay strictly `<` even with birthCanRetain: true. The gate reads
837
+ // the RAW birth time — calendar alignment applies only inside the bucket
838
+ // delta (retention_query_get_aligned_event_time_s at :1260).
839
+ //
840
+ // Compounded (retention_query.cpp:677-685): `rq->second = rq->first` — the
841
+ // return side IS the cohort side, so every cohort event is a return
842
+ // candidate ("DAU coming back").
843
+ //
844
+ // Unbounded modes:
845
+ // carryForward — read-time carry (:1854-1868): retained at N if active in
846
+ // ANY bucket ≤ N.
847
+ // carryBack — reverse-iteration carry (:274-278): retained at N if active
848
+ // in ANY bucket ≥ N.
849
+ // consecutiveForward — gated at WRITE time (:1275-1287): seen_in[N] is
850
+ // written only if seen_in[N−1] is already set (except N = 0). Since a
851
+ // user's returns arrive time-ordered, the surviving marks are exactly
852
+ // the maximal consecutive prefix {0..k} of buckets hit.
447
853
  //
448
- // We compute bucket = floor((return_ms - birth_ms) / DAY_MS) a raw ms-delta
449
- // from birth, NOT a UTC-calendar-day-number difference. So a return 23h after
450
- // birth lands in bucket 0; a return 25h after birth lands in bucket 1.
451
- //
452
- // Birth-can-retain (default false; retention_query.cpp:1097-1109):
453
- // if (birth_can_retain) return first_event_time_ms <= retention_event_time_ms;
454
- // else return first_event_time_ms < retention_event_time_ms;
455
- // We default to false (return events strictly after birth, ms-precise).
456
- //
457
- // Optional `carry_forward`: once retained on day M, count as retained on
458
- // every later bucket (Mixpanel's CARRY_FORWARD unbounded mode —
459
- // retention_query.cpp:1824-1837).
460
- //
461
- // Optional `segmentBy`: partition the cohort by the birth event's property
462
- // value (Mixpanel's segment_event=FIRST moderetention_query.cpp:1309).
463
- //
464
- // NOT IMPLEMENTED these are MORE common than initially documented; treat as
465
- // known scope gaps:
466
- // - COMPOUNDED retention (retention_query.cpp:670) reuses the first-event
467
- // filter as the return filter, making EVERY cohort event a retention
468
- // candidate. Used heavily in Mixpanel's "DAU coming back" reports.
469
- // - CARRY_BACK / CONSECUTIVE_FORWARD unbounded modes
470
- // - CALENDAR_START bucket alignment (retention_query.cpp:308-321)
471
- // - segment_event=SECOND (retention_query.cpp:1310 return event property)
472
- // - Cohort window (only users with birth in `from_date..to_date` are in
473
- // cohort; we use ALL users with the birth event in the dataset)
474
- // - week / month bucket units (only `day` here)
475
-
476
- const DAY_MS_RET = 86400 * 1000;
477
-
478
- function retention(events, { cohortEvent, returnEvent, dayBuckets = [1, 7, 14, 30], segmentBy, carry_forward = false, birthCanRetain = false, identityMap }) {
479
- if (!cohortEvent) throw new Error('retention requires cohortEvent');
480
- if (!returnEvent) throw new Error('retention requires returnEvent');
854
+ // calendarStart (retention_query.cpp:312-332, applied to the birth at :1260):
855
+ // ARB steps in fixed unit_s increments from the query interval start;
856
+ // production intervals start on unit boundaries, so the effect is flooring
857
+ // the BIRTH time to the bucket-unit boundary (week = ISO Monday, matching
858
+ // partitionByTimeBucket) before computing deltas. Month alignment floors to
859
+ // the calendar month start while the bucket WIDTH stays 31d fixed.
860
+ //
861
+ // Internal-event ignore list (retention_query.cpp:2546-2555): when a side has
862
+ // no explicit event selector (null / '$any_event'), $campaign_delivery,
863
+ // $campaign_bounced, $create_alias, $identify, $merge are ignored for that
864
+ // side. Explicit selectors bypass the list.
865
+ //
866
+ // segmentOn 'return' (SEGMENT_EVENT_SECOND, retention_query.cpp:1421-1444):
867
+ // the segment value is read from each RETURN event; a user joins a segment's
868
+ // cohort only via a qualifying return in that segment births are
869
+ // unsegmented unless birthCanRetain lets the birth itself qualify as a
870
+ // return (:1413-1419). ARB reads profile/SCD-style segment properties as-of
871
+ // the BIRTH time even in this mode (:1421-1444); this emulator segments on
872
+ // flat event properties only, so that read-path does not arise here.
873
+ //
874
+ // Unrecognized option keys THROW (1.6.0 behavior change — kills the
875
+ // silent-ignore class of bug where e.g. `compounded: true` was dropped).
876
+ //
877
+ // timeBucket composition caveat: the generic wrapper partitions EVENTS, so
878
+ // returns crossing a bucket edge are truncated. P1.6.5 re-anchors retention
879
+ // buckets on the BIRTH event time.
880
+
881
+ const RETENTION_UNIT_MS = {
882
+ hour: 3600 * 1000,
883
+ day: 86400 * 1000,
884
+ week: 7 * 86400 * 1000,
885
+ // "Maximum possible seconds in a month" — libquery/util.h:265-273.
886
+ month: 31 * 86400 * 1000,
887
+ };
888
+
889
+ const RETENTION_IGNORED_INTERNAL = new Set([
890
+ '$campaign_delivery', '$campaign_bounced', '$create_alias', '$identify', '$merge',
891
+ ]);
892
+
893
+ const RETENTION_KNOWN_KEYS = new Set([
894
+ 'type', 'profiles', 'identityMap', 'timeBucket', 'timeBucketRange',
895
+ 'cohortEvent', 'returnEvent', 'cohortWhere', 'returnWhere', 'compounded',
896
+ 'dayBuckets', 'bucketUnit', 'unbounded', 'carry_forward', 'bucketAlignment',
897
+ 'cohortWindow', 'segmentBy', 'segmentOn', 'birthCanRetain',
898
+ ]);
899
+
900
+ /**
901
+ * Does event `e` match one side (cohort or return) of the retention query?
902
+ * `sideEvent` null / '$any_event' = any event EXCEPT the internal ignore
903
+ * list (retention_query.cpp:2546-2555 — only applies when the side has no
904
+ * explicit selectors).
905
+ */
906
+ function retentionSideMatches(e, sideEvent, sideWhere) {
907
+ if (sideEvent == null || sideEvent === '$any_event') {
908
+ if (RETENTION_IGNORED_INTERNAL.has(e.event)) return false;
909
+ } else if (e.event !== sideEvent) {
910
+ return false;
911
+ }
912
+ return matchesWhere(e, sideWhere);
913
+ }
914
+
915
+ /**
916
+ * Floor `ms` to the start of its bucket unit (UTC). Week = ISO Monday,
917
+ * matching partitionByTimeBucket; month = calendar month start.
918
+ */
919
+ function retentionFloorToUnit(ms, unit) {
920
+ const DAY = RETENTION_UNIT_MS.day;
921
+ switch (unit) {
922
+ case 'hour': return ms - (ms % RETENTION_UNIT_MS.hour);
923
+ case 'day': return ms - (ms % DAY);
924
+ case 'week': {
925
+ const dayStart = ms - (ms % DAY);
926
+ const dow = new Date(dayStart).getUTCDay(); // 0 = Sunday
927
+ return dayStart - ((dow + 6) % 7) * DAY; // back to ISO Monday
928
+ }
929
+ case 'month': {
930
+ const d = new Date(ms);
931
+ return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1);
932
+ }
933
+ default: throw new Error(`retention: unknown bucketUnit "${unit}"`);
934
+ }
935
+ }
936
+
937
+ function retention(events, cfg) {
938
+ // Strict option keys — unknown keys were previously silently ignored
939
+ // (declared 1.6.0 behavior change).
940
+ for (const k of Object.keys(cfg)) {
941
+ if (!RETENTION_KNOWN_KEYS.has(k)) {
942
+ throw new Error(`retention: unrecognized option "${k}" (known: ${[...RETENTION_KNOWN_KEYS].join(', ')})`);
943
+ }
944
+ }
945
+
946
+ const {
947
+ cohortEvent = null,
948
+ cohortWhere,
949
+ compounded = false,
950
+ dayBuckets = [1, 7, 14, 30],
951
+ bucketUnit = 'day',
952
+ bucketAlignment = 'birth',
953
+ cohortWindow,
954
+ segmentBy,
955
+ segmentOn = 'birth',
956
+ birthCanRetain = false,
957
+ identityMap,
958
+ } = cfg;
959
+
960
+ if (!('cohortEvent' in cfg)) {
961
+ throw new Error('retention requires cohortEvent (null or "$any_event" = any event)');
962
+ }
963
+
964
+ // Compounded: return side := cohort side (retention_query.cpp:677-685).
965
+ let returnEvent, returnWhere;
966
+ if (compounded) {
967
+ if ('returnEvent' in cfg && cfg.returnEvent !== cohortEvent) {
968
+ throw new Error('retention: compounded sets returnEvent := cohortEvent — remove the conflicting returnEvent');
969
+ }
970
+ // Same guard for returnWhere (fix-round nit 4): `rq->second = rq->first`
971
+ // replaces the WHOLE return side, filters included — a conflicting
972
+ // returnWhere would be silently discarded, the exact silent-ignore bug
973
+ // class the strict-keys throw exists to kill.
974
+ if ('returnWhere' in cfg && cfg.returnWhere !== cohortWhere) {
975
+ throw new Error('retention: compounded sets returnWhere := cohortWhere — remove the conflicting returnWhere');
976
+ }
977
+ returnEvent = cohortEvent;
978
+ returnWhere = cohortWhere;
979
+ } else {
980
+ if (!('returnEvent' in cfg)) {
981
+ throw new Error('retention requires returnEvent (or compounded: true)');
982
+ }
983
+ returnEvent = cfg.returnEvent;
984
+ returnWhere = cfg.returnWhere;
985
+ }
986
+
481
987
  if (!Array.isArray(dayBuckets) || !dayBuckets.length) {
482
988
  throw new Error('retention requires non-empty dayBuckets');
483
989
  }
990
+ // Write horizon: the highest queried bucket index (see the bounds check
991
+ // at the mark loop — retention_query.cpp:1264).
992
+ const maxQueriedBucket = Math.max(...dayBuckets);
484
993
 
994
+ const unitMs = RETENTION_UNIT_MS[bucketUnit];
995
+ if (!unitMs) {
996
+ throw new Error(`retention: unknown bucketUnit "${bucketUnit}" (hour|day|week|month)`);
997
+ }
998
+
999
+ // `carry_forward` boolean is the deprecated alias for unbounded: 'carryForward'.
1000
+ let unbounded = cfg.unbounded;
1001
+ if (unbounded === undefined) unbounded = cfg.carry_forward ? 'carryForward' : 'none';
1002
+ if (!['none', 'carryForward', 'carryBack', 'consecutiveForward'].includes(unbounded)) {
1003
+ throw new Error(`retention: unknown unbounded mode "${unbounded}" (none|carryForward|carryBack|consecutiveForward)`);
1004
+ }
1005
+
1006
+ if (bucketAlignment !== 'birth' && bucketAlignment !== 'calendarStart') {
1007
+ throw new Error(`retention: unknown bucketAlignment "${bucketAlignment}" (birth|calendarStart)`);
1008
+ }
1009
+ if (segmentOn !== 'birth' && segmentOn !== 'return') {
1010
+ throw new Error(`retention: unknown segmentOn "${segmentOn}" (birth|return)`);
1011
+ }
1012
+
1013
+ let windowFromMs = -Infinity, windowToMs = Infinity;
1014
+ if (cohortWindow != null) {
1015
+ windowFromMs = toMs(cohortWindow.from);
1016
+ windowToMs = toMs(cohortWindow.to);
1017
+ if (!Number.isFinite(windowFromMs) || !Number.isFinite(windowToMs)) {
1018
+ throw new Error('retention: cohortWindow requires { from, to } timestamps');
1019
+ }
1020
+ }
1021
+
1022
+ const segmentedOnReturn = segmentOn === 'return' && !!segmentBy;
485
1023
  const userEvents = groupByUser(events, identityMap);
486
1024
 
487
- // segment → cohort users + per-user state
1025
+ // segment → cohort users + per-user retained-bucket marks
488
1026
  const cohorts = new Map();
489
1027
  const ensureSegment = (seg) => {
490
- if (!cohorts.has(seg)) cohorts.set(seg, { users: new Set(), birthMsByUser: new Map(), returnBucketsByUser: new Map() });
1028
+ if (!cohorts.has(seg)) cohorts.set(seg, { users: new Set(), markedByUser: new Map() });
491
1029
  return cohorts.get(seg);
492
1030
  };
493
1031
 
494
1032
  for (const [uid, evs] of userEvents) {
495
- // Birth = earliest cohortEvent for this user.
1033
+ // Birth = earliest cohort-side match INSIDE the cohort window
1034
+ // (inclusive bounds). ARB's in_birth_window gate
1035
+ // (retention_query.cpp:1188) means an out-of-window cohort event never
1036
+ // sets first_time (:1206-1208) — it does NOT consume the birth. A user
1037
+ // whose first-ever cohort event precedes the window still births at
1038
+ // their first in-window one. Under timeBucket the dispatcher runs each
1039
+ // bucket over the full stream with cohortWindow intersected to the
1040
+ // bucket bounds, so this in-window election is what realizes ARB's
1041
+ // per-(user, interval) retention history (:1334-1341): a user with
1042
+ // cohort events in two intervals re-births in each and appears in BOTH
1043
+ // cohorts, with returns bucketed relative to that interval's own birth.
496
1044
  const sorted = sortByTime(evs);
497
- const birth = sorted.find(e => e.event === cohortEvent);
1045
+ let birth = null, birthMs = NaN;
1046
+ for (const e of sorted) {
1047
+ if (!retentionSideMatches(e, cohortEvent, cohortWhere)) continue;
1048
+ const t = toMs(e.time);
1049
+ if (!Number.isFinite(t)) continue;
1050
+ if (t < windowFromMs || t > windowToMs) continue;
1051
+ birth = e;
1052
+ birthMs = t;
1053
+ break;
1054
+ }
498
1055
  if (!birth) continue;
499
- const birthMs = toMs(birth.time);
500
- if (!Number.isFinite(birthMs)) continue;
501
- const seg = segmentBy ? (birth[segmentBy] ?? 'unknown') : 'all';
502
- const sb = ensureSegment(seg);
503
- sb.users.add(uid);
504
- sb.birthMsByUser.set(uid, birthMs);
505
-
506
- const retBuckets = new Set();
507
- for (const ev of sorted) {
508
- if (ev.event !== returnEvent) continue;
509
- const evMs = toMs(ev.time);
1056
+
1057
+ const alignedBirthMs = bucketAlignment === 'calendarStart'
1058
+ ? retentionFloorToUnit(birthMs, bucketUnit)
1059
+ : birthMs;
1060
+
1061
+ // Collect this user's retained-bucket marks. Under segmentOn 'return'
1062
+ // each segment gets its own mark set (returns partition by their own
1063
+ // property value); otherwise one set under the birth-derived segment.
1064
+ const markedBySeg = new Map();
1065
+ const markSetFor = (seg) => {
1066
+ if (!markedBySeg.has(seg)) markedBySeg.set(seg, new Set());
1067
+ return markedBySeg.get(seg);
1068
+ };
1069
+ const birthSeg = segmentedOnReturn ? null
1070
+ : (segmentBy ? (birth[segmentBy] ?? 'unknown') : 'all');
1071
+
1072
+ for (const e of sorted) {
1073
+ if (!retentionSideMatches(e, returnEvent, returnWhere)) continue;
1074
+ const evMs = toMs(e.time);
510
1075
  if (!Number.isFinite(evMs)) continue;
511
- // Mixpanel ms-strict gate (retention_query.cpp:1097-1109).
512
- const passes = birthCanRetain ? (birthMs <= evMs) : (birthMs < evMs);
1076
+ // COR-233 gate: `<=` only when birthCanRetain AND this return also
1077
+ // matches the birth filter; raw (unaligned) birth time.
1078
+ const matchesFirst = retentionSideMatches(e, cohortEvent, cohortWhere);
1079
+ const passes = (birthCanRetain && matchesFirst) ? (birthMs <= evMs) : (birthMs < evMs);
513
1080
  if (!passes) continue;
514
- // Bucket by ms-delta Mixpanel time_to_retention_event_s / bucket_seconds.
515
- const bucket = Math.floor((evMs - birthMs) / DAY_MS_RET);
516
- if (bucket >= 0) retBuckets.add(bucket);
1081
+ const bucket = Math.floor((evMs - alignedBirthMs) / unitMs);
1082
+ if (bucket < 0) continue;
1083
+ // Write horizon (fix-round nit 3): ARB bounds-checks every write
1084
+ // against the queried bucket count — `bucket_index <
1085
+ // time_duration_buckets_num_buckets` (retention_query.cpp:1264) —
1086
+ // so a return past the last queried bucket is never recorded in
1087
+ // ANY mode. Without this, carryBack ("active in any bucket ≥ N")
1088
+ // would let an out-of-horizon return retro-mark every queried row.
1089
+ if (bucket > maxQueriedBucket) continue;
1090
+ const marked = markSetFor(segmentedOnReturn ? (e[segmentBy] ?? 'unknown') : birthSeg);
1091
+ // consecutiveForward write gate (retention_query.cpp:1275-1287):
1092
+ // bucket N marks only if N−1 already marked (except N = 0). Returns
1093
+ // arrive time-ordered, so marks form the maximal consecutive prefix.
1094
+ if (unbounded === 'consecutiveForward' && bucket !== 0 && !marked.has(bucket - 1)) continue;
1095
+ marked.add(bucket);
1096
+ }
1097
+
1098
+ if (segmentedOnReturn) {
1099
+ // Births are unsegmented under segmentOn 'return' — a user joins a
1100
+ // segment's cohort only via a qualifying return in that segment
1101
+ // (retention_query.cpp:1413-1419; birthCanRetain lets the birth
1102
+ // itself qualify when it matches the return side).
1103
+ for (const [seg, marked] of markedBySeg) {
1104
+ const sb = ensureSegment(seg);
1105
+ sb.users.add(uid);
1106
+ sb.markedByUser.set(uid, marked);
1107
+ }
1108
+ } else {
1109
+ const sb = ensureSegment(birthSeg);
1110
+ sb.users.add(uid);
1111
+ sb.markedByUser.set(uid, markedBySeg.get(birthSeg) || new Set());
517
1112
  }
518
- sb.returnBucketsByUser.set(uid, retBuckets);
519
1113
  }
520
1114
 
521
1115
  const out = [];
@@ -524,18 +1118,20 @@ function retention(events, { cohortEvent, returnEvent, dayBuckets = [1, 7, 14, 3
524
1118
  for (const day of dayBuckets) {
525
1119
  let retained = 0;
526
1120
  for (const uid of sb.users) {
527
- const buckets = sb.returnBucketsByUser.get(uid);
528
- if (!buckets) continue;
529
- if (carry_forward) {
530
- // Retained on bucket N if hit any bucket in [0, N] (or [1, N] if you exclude bucket 0).
531
- let hit = false;
532
- for (const b of buckets) {
533
- if (b <= day) { hit = true; break; }
534
- }
535
- if (hit) retained++;
1121
+ const marked = sb.markedByUser.get(uid);
1122
+ if (!marked || !marked.size) continue;
1123
+ let hit = false;
1124
+ if (unbounded === 'carryForward') {
1125
+ // Read-time carry (:1854-1868): active in ANY bucket ≤ N.
1126
+ for (const b of marked) { if (b <= day) { hit = true; break; } }
1127
+ } else if (unbounded === 'carryBack') {
1128
+ // Reverse-iteration carry (:274-278): active in ANY bucket ≥ N.
1129
+ for (const b of marked) { if (b >= day) { hit = true; break; } }
536
1130
  } else {
537
- if (buckets.has(day)) retained++;
1131
+ // 'none' + 'consecutiveForward' (write-gated) read membership.
1132
+ hit = marked.has(day);
538
1133
  }
1134
+ if (hit) retained++;
539
1135
  }
540
1136
  out.push({
541
1137
  day,
@@ -554,16 +1150,25 @@ function retention(events, { cohortEvent, returnEvent, dayBuckets = [1, 7, 14, 3
554
1150
  //
555
1151
  // Reference: backend/arb/reader/queries/session_query.cpp.
556
1152
  //
557
- // Mixpanel computes sessions at query time using a 30-min gap (default) +
558
- // 24h max model and emits synthetic event properties: $duration_s,
559
- // $event_count, $origin_start, $origin_end. Our generator pre-stamps
560
- // `session_id` using the same rules; here we trust that stamping and just
561
- // group aggregate per session.
1153
+ // Mixpanel computes sessions at QUERY TIME (30-min gap default + 24h max +
1154
+ // UTC day-boundary split) sessions are never stored. v1.6.0 (P1.7.2)
1155
+ // aligns: `source: 'derived'` (default) re-derives sessions from timestamps
1156
+ // via sessionize(); `source: 'stamped'` keeps the v1.5 behavior of trusting
1157
+ // generator-stamped `session_id` (events without one are excluded there).
1158
+ // Declared v1.6.0 behavior change: the default no longer reads session_id
1159
+ // at all, so unstamped events now participate.
1160
+ //
1161
+ // When any event carries a stamped session_id, every row also reports
1162
+ // `stampedDivergence`: the number of events whose stamped session BOUNDARY
1163
+ // disagrees with the derived one — event i (per user, time-sorted) diverges
1164
+ // when `stamped(i) === stamped(i-1)` and `derived(i) === derived(i-1)`
1165
+ // disagree. Pairs where either event lacks a stamp are skipped (no boundary
1166
+ // to compare). This makes the generator-vs-query-time seam measurable.
562
1167
  //
563
1168
  // Returns an array with one row per requested metric:
564
- // [{ metric: 'count', avg, median, p90, total_sessions }]
565
- // [{ metric: 'duration', avg_ms, median_ms, p90_ms, total_sessions }]
566
- // [{ metric: 'eventsPerSession',avg, median, p90, total_sessions }]
1169
+ // [{ metric: 'count', avg, median, p90, total_sessions, source, stampedDivergence }]
1170
+ // [{ metric: 'duration', avg_ms, median_ms, p90_ms, total_sessions, source, stampedDivergence }]
1171
+ // [{ metric: 'eventsPerSession',avg, median, p90, total_sessions, source, stampedDivergence }]
567
1172
 
568
1173
  // ── COUNT_DISTINCT(property) — v1.5.1 ─────────────────────────────────────
569
1174
  //
@@ -582,37 +1187,508 @@ function distinctCount(events, { property, event, topN = 25 }) {
582
1187
  return [result];
583
1188
  }
584
1189
 
585
- function sessionMetrics(events, { event, metrics = ['count', 'duration', 'eventsPerSession'], identityMap }) {
586
- const userEvents = groupByUser(events, identityMap);
587
- const sessionsByUser = new Map();
588
- for (const [uid, evs] of userEvents) {
589
- // One session bucket per (user, session_id). Events without session_id
590
- // are excluded — Mixpanel only emits session reports for events that
591
- // landed inside an evaluated session.
592
- const buckets = new Map();
593
- for (const ev of evs) {
594
- if (ev.session_id == null) continue;
595
- const sid = String(ev.session_id);
596
- if (!buckets.has(sid)) buckets.set(sid, []);
597
- buckets.get(sid).push(ev);
598
- }
599
- // Optional event filter: only sessions containing this event qualify.
600
- if (event) {
601
- for (const [sid, evs2] of [...buckets]) {
602
- if (!evs2.some(e => e.event === event)) buckets.delete(sid);
1190
+ // ── Event Breakdown (Insights: TOTAL events segmented by a property) ──
1191
+ //
1192
+ // The most common Mixpanel report shape: count of EVENTS per property value
1193
+ // (not per-user grouping that's aggregatePerUser). Semantics from
1194
+ // normal_query.cpp ACTION_TYPE_FOR_EACH (:1718-1776) + coerce.js rulebooks:
1195
+ //
1196
+ // - List-valued property → the event counts once PER list item (an event
1197
+ // with ["a","b"] increments both segments; recursion in
1198
+ // query_record_result over list_cursor items).
1199
+ // - Empty list literal segment "$empty_list" (normal_query.cpp:1762
1200
+ // value_create_string("$empty_list")).
1201
+ // - null AND undefined both → the "undefined" bucket (string-typecast
1202
+ // default, arb_selector.py:889-916; normal_query.cpp:1769-1773 routes
1203
+ // VALUE_TYPE_UNDEFINED|NULL to the undefined inner action).
1204
+ // - Segment identity is case-SENSITIVE and type-tagged (hash_value.c raw
1205
+ // XXH3 string hash + per-type tag mixins; cmp.c arb_strcmp) — "iOS" and
1206
+ // "ios" are separate rows; number 1 and string "1" are separate rows.
1207
+ // - Sort count desc, keep top `topN` (default 250 — normal_query.cpp
1208
+ // :1195-1197 "If no meaningful limit is supplied, set it to 250"), and
1209
+ // TRUNCATE: no "other" rollup bucket (query_set_top_results marks the top
1210
+ // N leaf results; the rest simply don't render — normal_query.cpp
1211
+ // :1865-1905).
1212
+ // - v1.6.0 (P1.7.3) `countType: 'sessions'`: an event counts at most once
1213
+ // per (user, session, segment) — counted iff the segment has no prior
1214
+ // counted event in that session, then stamp
1215
+ // (query_record_per_user_session_segment_state, normal_query.cpp
1216
+ // :1318-1352 — the per-segment state timestamp is compared against the
1217
+ // user's CURRENT session start and updated on count). Sessions are
1218
+ // derived via sessionize() from the FULL event stream: ARB updates
1219
+ // session state on every event and only gates result RECORDING on the
1220
+ // filter (:2271-2280). `sessionTimeoutMs`/`maxSessionMs` thread through.
1221
+ // In sessions mode, `total_users` counts users of COUNTED events only.
1222
+ // - `countType: 'unique'`: the per-segment metric is distinct resolved
1223
+ // users (COUNT_TYPE_UNIQUE — query_record_per_user_state keeps a
1224
+ // per-(segment) user-state container, normal_query.cpp:1300-1316):
1225
+ // `count` equals `total_users`, and the topN cut sorts on that metric,
1226
+ // so a many-events/few-users segment can rank below a few-events/
1227
+ // many-users one. Unrecognized countType values THROW (same
1228
+ // strict-option rule as retention keys — a typo must not silently
1229
+ // fall back to general counting).
1230
+ //
1231
+ // Returns rows: `{ value, count, total_users }` where total_users is the
1232
+ // distinct identity-resolved user count within the segment.
1233
+
1234
+ function eventBreakdown(events, { event, breakdownProperty, topN = 250, countType, sessionTimeoutMs, maxSessionMs, firstTimeOnly = false, identityMap }) {
1235
+ if (!breakdownProperty) throw new Error('eventBreakdown requires `breakdownProperty`');
1236
+ if (countType !== undefined && countType !== 'general' && countType !== 'unique' && countType !== 'sessions') {
1237
+ throw new Error(`eventBreakdown: countType must be "general", "unique", or "sessions", got "${countType}"`);
1238
+ }
1239
+ // Derive BEFORE any event filtering (normal_query.cpp:2271-2280 — session
1240
+ // state updates are unconditional; the name/firstTime filter only gates
1241
+ // recording). Safe under the timeBucket wrapper's per-bucket slices:
1242
+ // sessions never cross UTC midnight (daySplit, sessionize.js) and bucket
1243
+ // units cut at midnights, so slice-derived boundaries equal full-stream
1244
+ // boundaries (B5 invariant tests).
1245
+ let sessionOf = null;
1246
+ if (countType === 'sessions') {
1247
+ const { sessions } = sessionize(events, {
1248
+ timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
1249
+ maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
1250
+ identityMap,
1251
+ });
1252
+ sessionOf = new Map();
1253
+ sessions.forEach((s, i) => { for (const e of s.events) sessionOf.set(e, i); });
1254
+ }
1255
+ if (firstTimeOnly) {
1256
+ // P1.4: restrict to each user's first-ever occurrence of `event`
1257
+ // before segmenting (the $nth_time_event rewrite — first-time.js).
1258
+ events = filterFirstTimeEver(events, { event, identityMap });
1259
+ }
1260
+ const segments = new Map(); // type-tagged segment key → { value, count, users, sessions }
1261
+ const record = (rawValue, e) => {
1262
+ const key = breakdownSegmentKey(rawValue);
1263
+ let seg = segments.get(key);
1264
+ if (!seg) {
1265
+ seg = { value: coerceToBreakdownKey(rawValue), count: 0, users: new Set(), sessions: sessionOf ? new Set() : null };
1266
+ segments.set(key, seg);
1267
+ }
1268
+ if (sessionOf) {
1269
+ const si = sessionOf.get(e);
1270
+ if (si === undefined) return; // no derived session (unresolvable id / bad time)
1271
+ if (seg.sessions.has(si)) return; // already counted this session for this segment
1272
+ seg.sessions.add(si); // stamp (normal_query.cpp:1330-1339)
1273
+ }
1274
+ seg.count++;
1275
+ seg.users.add(userIdOf(e, identityMap));
1276
+ };
1277
+ for (const e of events) {
1278
+ if (!e || (event && e.event !== event)) continue;
1279
+ const v = e[breakdownProperty];
1280
+ if (Array.isArray(v)) {
1281
+ if (v.length === 0) record('$empty_list', e);
1282
+ else for (const item of v) record(item, e);
1283
+ } else {
1284
+ record(v, e);
1285
+ }
1286
+ }
1287
+ const unique = countType === 'unique';
1288
+ return [...segments.values()]
1289
+ .map(s => ({ value: s.value, count: unique ? s.users.size : s.count, total_users: s.users.size }))
1290
+ .sort((a, b) => b.count - a.count)
1291
+ .slice(0, topN);
1292
+ }
1293
+
1294
+ // ── Uniques (Insights: unique users per interval / XAU / cumulative) ──
1295
+ //
1296
+ // COUNT_TYPE_UNIQUE semantics from normal_query.cpp:
1297
+ // - Per-interval dedup is INDEPENDENT per bucket (query_record_per_user_state
1298
+ // keeps a per-(segment, interval) user-state container — :1300-1316): a user
1299
+ // active on 3 days counts 3 in a daily series. `unit: 'range'` is the same
1300
+ // machinery over a single interval (segmentation_arb.py:933 "total uniques").
1301
+ // - Rolling look-back window (WAU/MAU — window_length ≠ 0 branch, :1797-1830):
1302
+ // an event contributes to every interval whose decision post (interval END
1303
+ // for look-back windows) lies within [event_time, event_time + W], i.e. an
1304
+ // event on day E lands the user in daily buckets [E, E + W − 1]. XAU is a
1305
+ // look-back window over daily intervals — NOT a calendar week/month.
1306
+ // - Cumulative (accumulate_uniques_result, :1834-1863): each interval's count
1307
+ // becomes the size of the running distinct-id set through that interval.
1308
+ // - Events whose resolved user id is EMPTY are skipped (normal_query.cpp
1309
+ // :2200-2208 — distinct_id[0] == '\0' → return 0).
1310
+ // - v1.6.0 (P1.7.3) `countType: 'sessions'`: the bucket value is the count
1311
+ // of distinct (user, session) pairs active in the bucket — NOT distinct
1312
+ // users (per-interval result_values each own a user_session_states
1313
+ // container, normal_query.cpp:397-417; recording dedups per session via
1314
+ // query_record_per_user_session_segment_state, :1318-1352, dispatched at
1315
+ // :1462-1464). Sessions derive via sessionize() from the FULL stream
1316
+ // (state updates are unconditional, :2271-2280). Composes with
1317
+ // `rollingWindow` (the window branch at :1797-1830 routes through the
1318
+ // same count-type dispatch) but NOT `cumulative` — ARB expresses
1319
+ // cumulative only as the distinct COUNT_TYPE_CUMULATIVE_UNIQUE enum
1320
+ // (accumulate_uniques_result gate at :1860), so there is no cumulative
1321
+ // sessions count.
1322
+ //
1323
+ // Returns rows: `{ period, uniques }` sorted by period.
1324
+
1325
+ function uniques(events, {
1326
+ event,
1327
+ where,
1328
+ unit = 'day',
1329
+ rollingWindow,
1330
+ cumulative = false,
1331
+ countType,
1332
+ sessionTimeoutMs,
1333
+ maxSessionMs,
1334
+ firstTimeOnly = false,
1335
+ identityMap,
1336
+ }) {
1337
+ let sessionOf = null;
1338
+ if (countType === 'sessions') {
1339
+ if (cumulative) {
1340
+ throw new Error('uniques: cumulative sessions is not supported (ARB has no cumulative sessions count type — normal_query.cpp:1860)');
1341
+ }
1342
+ // Derive BEFORE the event/where filter (normal_query.cpp:2271-2280).
1343
+ const { sessions } = sessionize(events, {
1344
+ timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
1345
+ maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
1346
+ identityMap,
1347
+ });
1348
+ sessionOf = new Map();
1349
+ sessions.forEach((s, i) => { for (const e of s.events) sessionOf.set(e, i); });
1350
+ }
1351
+ if (rollingWindow != null && cumulative) {
1352
+ throw new Error('uniques: rollingWindow and cumulative are mutually exclusive');
1353
+ }
1354
+ if (rollingWindow != null && (!Number.isInteger(rollingWindow) || rollingWindow < 1)) {
1355
+ throw new Error('uniques: rollingWindow must be a positive integer');
1356
+ }
1357
+ if (!['day', 'week', 'month', 'range'].includes(unit)) {
1358
+ throw new Error(`uniques: unknown unit "${unit}"`);
1359
+ }
1360
+
1361
+ if (firstTimeOnly) {
1362
+ // P1.4: `where` acts as the PRE-filter — it defines the universe the
1363
+ // per-user first is picked from. The rewritten query does not re-apply
1364
+ // it (see first-time.js header), so the loop below must not either.
1365
+ events = filterFirstTimeEver(events, { event, preWhere: where, identityMap });
1366
+ }
1367
+
1368
+ // `key` is the per-bucket dedup identity: the resolved user id, or the
1369
+ // derived session index in sessions mode (a session belongs to one user,
1370
+ // so the index IS the (user, session) pair).
1371
+ const filtered = [];
1372
+ for (const e of events) {
1373
+ if (!e || (event && e.event !== event)) continue;
1374
+ if (!firstTimeOnly && !matchesWhere(e, where)) continue;
1375
+ const uid = userIdOf(e, identityMap);
1376
+ if (uid === undefined || uid === null || uid === '') continue; // :2200-2208
1377
+ const key = sessionOf ? sessionOf.get(e) : uid;
1378
+ if (key === undefined) continue; // sessions mode: no derived session (bad time)
1379
+ filtered.push({ e, key });
1380
+ }
1381
+
1382
+ if (rollingWindow != null) {
1383
+ // Daily intervals only (unit forced to 'day'); W-day look-back per bucket.
1384
+ const W = rollingWindow;
1385
+ const DAY = 86400_000;
1386
+ let minDay = Infinity, maxDay = -Infinity;
1387
+ /** @type {Array<[number, string|number]>} */
1388
+ const eventDays = []; // [dayIndex, key]
1389
+ for (const { e, key } of filtered) {
1390
+ const ms = toMs(e.time);
1391
+ if (!Number.isFinite(ms)) continue;
1392
+ const d = Math.floor(ms / DAY);
1393
+ eventDays.push([d, key]);
1394
+ if (d < minDay) minDay = d;
1395
+ if (d > maxDay) maxDay = d;
1396
+ }
1397
+ if (!eventDays.length) return [];
1398
+ const perBucket = new Map(); // dayIndex → Set<key>
1399
+ for (let d = minDay; d <= maxDay; d++) perBucket.set(d, new Set());
1400
+ for (const [d, key] of eventDays) {
1401
+ // Event on day E → buckets [E, E + W − 1], clamped to the observed range.
1402
+ // Exact-midnight boundary (fix-round nit 6, REFUTED): ARB's decision
1403
+ // check is end-post inclusive (normal_query.cpp:1825 `post >= time_min`,
1404
+ // and end(E−1) == an exact-midnight event_time), which would admit day
1405
+ // E−1 — but the interval scan STARTS at
1406
+ // `interval_min = uniform_intervals_for_relaxed(time_min)` (:1801), and
1407
+ // a fencepost timestamp maps to the LATER interval (integer division,
1408
+ // uniform_intervals.h:57), so day E−1 is never evaluated. Upper edge:
1409
+ // end(E+W−1) == time_max for a midnight event → inclusive → included.
1410
+ // Net: [E, E+W−1] for every event, midnight or not — exactly this loop.
1411
+ const hi = Math.min(d + W - 1, maxDay);
1412
+ for (let b = d; b <= hi; b++) perBucket.get(b).add(key);
1413
+ }
1414
+ return [...perBucket.entries()].map(([d, set]) => ({
1415
+ period: new Date(d * DAY).toISOString().slice(0, 10),
1416
+ uniques: set.size,
1417
+ }));
1418
+ }
1419
+
1420
+ if (unit === 'range') {
1421
+ const all = new Set(filtered.map(({ key }) => key));
1422
+ return [{ period: 'range', uniques: all.size }];
1423
+ }
1424
+
1425
+ const buckets = partitionByTimeBucket(filtered.map(({ e }) => e), /** @type {*} */ (unit));
1426
+ // Re-resolve key per event inside each bucket via a lookup built above —
1427
+ // avoid resolving twice by mapping event object → key.
1428
+ const keyOf = new Map(filtered.map(({ e, key }) => [e, key]));
1429
+ const running = new Set();
1430
+ return buckets.map(({ period, events: evs }) => {
1431
+ const set = new Set();
1432
+ for (const e of evs) set.add(keyOf.get(e));
1433
+ if (cumulative) {
1434
+ for (const u of set) running.add(u);
1435
+ return { period, uniques: running.size };
1436
+ }
1437
+ return { period, uniques: set.size };
1438
+ });
1439
+ }
1440
+
1441
+ // ── Lifecycle (P1.8 — the LCA board template classification) ──
1442
+ //
1443
+ // Mixpanel has NO engine lifecycle query (query_type.cpp:8-30 enumerates
1444
+ // every query type — nothing lifecycle-shaped). "Lifecycle" ships as the
1445
+ // Lifecycle Cohort Analysis board template: Insights uniques filtered by
1446
+ // four behavioral cohorts on a Value Moment event, in 7- and 30-day period
1447
+ // variants (iron/common/report/dashboards/types.ts:367-390 —
1448
+ // LifecycleCohortAnalysisTemplateFields.VALUE_MOMENT is the one required
1449
+ // template field). The canonical in-source cohort definition is
1450
+ // weeklyResurrectedUserBookmark
1451
+ // (iron/common/widgets/profile-summary/bookmark_templates.ts:179-320):
1452
+ // resurrected = value-moment count AtLeast 1 in the last 30d AND EqualTo 0
1453
+ // in 60→30d ago AND AtLeast 1 in 90→60d ago. Generalized to period P:
1454
+ // active in T, inactive in T−1, active in some period before T−1.
1455
+ //
1456
+ // ARB-DIVERGENCE (Resurrected lookback — fix-round Q6, KEEP): the bookmark's
1457
+ // third clause looks back exactly ONE period past the dormancy gap (AtLeast 1
1458
+ // in 90→60d ago = active in T−2 specifically). This emulator accepts activity
1459
+ // in ANY period before T−1 — growth accounting: every active user is exactly
1460
+ // one of New/Retained/Resurrected, so new + retained + resurrected = period
1461
+ // uniques and the classes tile the actives with no unclassified remainder.
1462
+ // Under the literal bookmark, a returner whose last activity predates T−2
1463
+ // falls out of ALL four cohorts (the LCA board undercounts long-dormant
1464
+ // returners by construction). Dungeon stories assert on the growth-accounting
1465
+ // partition; anyone reconciling against a real LCA board should expect
1466
+ // `resurrected` here ≥ the board's whenever dormancy gaps exceed one period.
1467
+ //
1468
+ // DECLARED DIVERGENCE (tiled vs rolling): the real template's cohort
1469
+ // windows are rolling, re-anchored as-of each charting interval
1470
+ // (api/version_2_0/insights/params.py:1396-1403
1471
+ // generate_behaviors_as_of_project_times_for_cohorts →
1472
+ // behaviors/count.py:213-300 behavior_as_of_project_times;
1473
+ // cohort_count_query.cpp:48-59 — per-interval user-state containers with an
1474
+ // associated_time). Tiling fixed periods back from the dataset's LAST EVENT
1475
+ // DAY is the deterministic equivalent for generated fixture data: identical
1476
+ // classification rules, stable period edges. Rows span from the first
1477
+ // value-moment event day through the dataset's last event day; the earliest
1478
+ // tile may be partial, and its actives are all New by dataset-boundedness
1479
+ // (the real template's ~5y first-time lookback always covers a dungeon
1480
+ // window — see first-time.js header).
1481
+ //
1482
+ // The dormancy test is an EqualTo-0 filter (bookmark_templates.ts) — ONE
1483
+ // stray value-moment event inside a would-be dormancy window reclassifies
1484
+ // the user (resurrected → retained). Gap discipline is therefore a
1485
+ // story-authoring requirement (P2.3 atom).
1486
+ function lifecycle(events, { valueMomentEvent, periodDays = 7, identityMap }) {
1487
+ if (!valueMomentEvent) throw new Error('lifecycle requires valueMomentEvent');
1488
+ if (periodDays !== 7 && periodDays !== 30) {
1489
+ throw new Error(`lifecycle: periodDays must be 7 or 30 (the LCA template variants), got ${periodDays}`);
1490
+ }
1491
+ const DAY = 86400_000;
1492
+ const P = periodDays;
1493
+
1494
+ // Anchor period edges on the FULL stream's last event day — the dataset
1495
+ // boundary, not the value-moment subset: a value moment that stops firing
1496
+ // must still be reported dormant through the end of the data.
1497
+ let lastDay = -Infinity;
1498
+ for (const e of events) {
1499
+ const ms = toMs(e && e.time);
1500
+ if (Number.isFinite(ms)) {
1501
+ const d = Math.floor(ms / DAY);
1502
+ if (d > lastDay) lastDay = d;
1503
+ }
1504
+ }
1505
+
1506
+ // "Active" = did the value-moment event; other events never qualify
1507
+ // (the template's cohorts all filter on the one Value Moment —
1508
+ // bookmark_templates.ts custom-property behaviors).
1509
+ const vm = [];
1510
+ let firstDay = Infinity;
1511
+ for (const e of events) {
1512
+ if (!e || e.event !== valueMomentEvent) continue;
1513
+ const uid = userIdOf(e, identityMap);
1514
+ if (uid === undefined || uid === null || uid === '') continue; // normal_query.cpp:2200-2208
1515
+ const ms = toMs(e.time);
1516
+ if (!Number.isFinite(ms)) continue;
1517
+ const day = Math.floor(ms / DAY);
1518
+ vm.push({ uid, day });
1519
+ if (day < firstDay) firstDay = day;
1520
+ }
1521
+ if (!vm.length) return [];
1522
+
1523
+ // Period i (ascending) covers day indexes (endDay(i) − P, endDay(i)],
1524
+ // endDay(i) = lastDay − (numPeriods − 1 − i) · P.
1525
+ const numPeriods = Math.ceil((lastDay - firstDay + 1) / P);
1526
+ const periodOf = (day) => numPeriods - 1 - Math.floor((lastDay - day) / P);
1527
+
1528
+ const activity = new Map(); // uid → Set<periodIdx>
1529
+ for (const { uid, day } of vm) {
1530
+ let set = activity.get(uid);
1531
+ if (!set) activity.set(uid, set = new Set());
1532
+ set.add(periodOf(day));
1533
+ }
1534
+
1535
+ // New = first-ever value moment falls in T. LCA's New cohort rides the
1536
+ // $nth_time_event rewrite, so reuse P1.4's machinery (first-time.js);
1537
+ // its both-pass timestamp-tie edge is harmless here — same user, same
1538
+ // timestamp, same period.
1539
+ const firstPeriod = new Map(); // uid → periodIdx
1540
+ for (const e of filterFirstTimeEver(events, { event: valueMomentEvent, identityMap })) {
1541
+ const uid = userIdOf(e, identityMap);
1542
+ const ms = toMs(e.time);
1543
+ if (uid === undefined || uid === null || uid === '' || !Number.isFinite(ms)) continue;
1544
+ if (!firstPeriod.has(uid)) firstPeriod.set(uid, periodOf(Math.floor(ms / DAY)));
1545
+ }
1546
+
1547
+ const rows = [];
1548
+ for (let i = 0; i < numPeriods; i++) {
1549
+ const endDay = lastDay - (numPeriods - 1 - i) * P;
1550
+ const row = {
1551
+ period: new Date(endDay * DAY).toISOString().slice(0, 10),
1552
+ new: 0,
1553
+ retained: 0,
1554
+ resurrected: 0,
1555
+ dormant: 0,
1556
+ };
1557
+ for (const [uid, set] of activity) {
1558
+ const active = set.has(i);
1559
+ const activePrev = i > 0 && set.has(i - 1);
1560
+ if (active) {
1561
+ // New takes precedence: a first-ever event can't retain or
1562
+ // resurrect (no prior activity by definition), so the branch
1563
+ // order below is belt-and-braces — but explicit per spec.
1564
+ if (firstPeriod.get(uid) === i) {
1565
+ row.new++;
1566
+ } else if (activePrev) {
1567
+ row.retained++;
1568
+ } else {
1569
+ // Resurrected needs activity in some period BEFORE the
1570
+ // (inactive) T−1 — ANY prior period, not the bookmark's
1571
+ // exact T−2 (see ARB-DIVERGENCE header: growth
1572
+ // accounting keeps the partition exhaustive). Non-first
1573
+ // active with inactive T−1 always has one, so this scan
1574
+ // can't miss — kept as a real check rather than an
1575
+ // assumption.
1576
+ for (const j of set) {
1577
+ if (j < i - 1) { row.resurrected++; break; }
1578
+ }
1579
+ }
1580
+ } else if (activePrev) {
1581
+ // EqualTo-0 in T, AtLeast-1 in T−1 — dormant counts exactly
1582
+ // one period, then the user drops out of every class until
1583
+ // (unless) they resurrect. Reported positive; charting it
1584
+ // negative is a frontend concern.
1585
+ row.dormant++;
1586
+ }
1587
+ }
1588
+ rows.push(row);
1589
+ }
1590
+ return rows;
1591
+ }
1592
+
1593
+ // ── Top Paths (P1.9 — the Flows report) ──
1594
+ //
1595
+ // Thin dispatch over the flows module: extractFlows runs the per-user ARB
1596
+ // state machine (anchors, buffers, hidden/visible, collapse, count types),
1597
+ // aggregateFlows builds + prunes the prefix tree and shapes the list/sankey
1598
+ // output. All ARB citations live in flows.js.
1599
+ function topPaths(events, {
1600
+ anchors, forward, reverse, countType, hiddenEvents, visibleEvents,
1601
+ collapseRepeated, breakdownProperty, sessionTimeoutMs, maxSessionMs,
1602
+ identityMap, output, cardinalityThreshold, listSortPriority,
1603
+ }) {
1604
+ const flows = extractFlows(events, {
1605
+ anchors, forward, reverse, countType, hiddenEvents, visibleEvents,
1606
+ collapseRepeated, breakdownProperty, sessionTimeoutMs, maxSessionMs,
1607
+ identityMap,
1608
+ });
1609
+ const opts = { output, cardinalityThreshold };
1610
+ if (listSortPriority !== undefined) /** @type {*} */ (opts).listSortPriority = listSortPriority;
1611
+ return aggregateFlows(flows, opts);
1612
+ }
1613
+
1614
+ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'eventsPerSession'], source = 'derived', sessionTimeoutMs, maxSessionMs, identityMap }) {
1615
+ if (source !== 'derived' && source !== 'stamped') {
1616
+ throw new Error(`sessionMetrics source must be 'derived' or 'stamped', got '${source}'`);
1617
+ }
1618
+ // Query-time derivation runs in BOTH modes: it is the metric source when
1619
+ // source==='derived', and the divergence reference when source==='stamped'.
1620
+ const { sessions } = sessionize(events, {
1621
+ timeoutMs: sessionTimeoutMs ?? 30 * 60_000,
1622
+ maxSessionMs: maxSessionMs ?? 24 * 3_600_000,
1623
+ identityMap,
1624
+ });
1625
+
1626
+ // stampedDivergence — boundary disagreements between consecutive stamped
1627
+ // events (see block comment above). Derived membership is by object
1628
+ // reference: sessionize partitions each user's events, so `sidOf` is total
1629
+ // over resolvable events.
1630
+ const hasStamps = events.some(e => e && e.session_id != null);
1631
+ let stampedDivergence = null;
1632
+ if (hasStamps) {
1633
+ const sidOf = new Map();
1634
+ sessions.forEach((s, i) => { for (const e of s.events) sidOf.set(e, i); });
1635
+ stampedDivergence = 0;
1636
+ for (const [, evs] of groupByUser(events, identityMap)) {
1637
+ const sorted = sortByTime(evs);
1638
+ for (let i = 1; i < sorted.length; i++) {
1639
+ const a = sorted[i - 1], b = sorted[i];
1640
+ if (a.session_id == null || b.session_id == null) continue;
1641
+ const stampedSame = String(a.session_id) === String(b.session_id);
1642
+ const derivedSame = sidOf.get(a) !== undefined && sidOf.get(a) === sidOf.get(b);
1643
+ if (stampedSame !== derivedSame) stampedDivergence++;
603
1644
  }
604
1645
  }
605
- if (buckets.size) sessionsByUser.set(uid, buckets);
606
1646
  }
1647
+
607
1648
  const allSessions = []; // { duration_ms, event_count }
608
1649
  const sessionCountsPerUser = [];
609
- for (const [, buckets] of sessionsByUser) {
610
- sessionCountsPerUser.push(buckets.size);
611
- for (const [, evs] of buckets) {
612
- const sorted = sortByTime(evs);
613
- const start = toMs(sorted[0].time);
614
- const end = toMs(sorted[sorted.length - 1].time);
615
- allSessions.push({ duration_ms: end - start, event_count: sorted.length });
1650
+ if (source === 'derived') {
1651
+ // Optional event filter: only sessions containing this event qualify
1652
+ // (session.events holds references to the original records).
1653
+ const qualifying = event
1654
+ ? sessions.filter(s => s.events.some(e => e && e.event === event))
1655
+ : sessions;
1656
+ const perUser = new Map();
1657
+ for (const s of qualifying) {
1658
+ perUser.set(s.userId, (perUser.get(s.userId) || 0) + 1);
1659
+ // endMs is stamped at the session's LAST event (idle tail excluded),
1660
+ // so endMs - startMs matches $duration_s at ms resolution.
1661
+ allSessions.push({ duration_ms: s.endMs - s.startMs, event_count: s.event_count });
1662
+ }
1663
+ sessionCountsPerUser.push(...perUser.values());
1664
+ } else {
1665
+ // v1.5 stamped path: one session bucket per (user, session_id); events
1666
+ // without session_id are excluded.
1667
+ const userEvents = groupByUser(events, identityMap);
1668
+ const sessionsByUser = new Map();
1669
+ for (const [uid, evs] of userEvents) {
1670
+ const buckets = new Map();
1671
+ for (const ev of evs) {
1672
+ if (ev.session_id == null) continue;
1673
+ const sid = String(ev.session_id);
1674
+ if (!buckets.has(sid)) buckets.set(sid, []);
1675
+ buckets.get(sid).push(ev);
1676
+ }
1677
+ if (event) {
1678
+ for (const [sid, evs2] of [...buckets]) {
1679
+ if (!evs2.some(e => e.event === event)) buckets.delete(sid);
1680
+ }
1681
+ }
1682
+ if (buckets.size) sessionsByUser.set(uid, buckets);
1683
+ }
1684
+ for (const [, buckets] of sessionsByUser) {
1685
+ sessionCountsPerUser.push(buckets.size);
1686
+ for (const [, evs] of buckets) {
1687
+ const sorted = sortByTime(evs);
1688
+ const start = toMs(sorted[0].time);
1689
+ const end = toMs(sorted[sorted.length - 1].time);
1690
+ allSessions.push({ duration_ms: end - start, event_count: sorted.length });
1691
+ }
616
1692
  }
617
1693
  }
618
1694
  const out = [];
@@ -624,6 +1700,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
624
1700
  median: median(sessionCountsPerUser),
625
1701
  p90: percentile(sessionCountsPerUser, 0.9),
626
1702
  total_sessions: allSessions.length,
1703
+ source,
1704
+ stampedDivergence,
627
1705
  });
628
1706
  }
629
1707
  if (requested.has('duration')) {
@@ -634,6 +1712,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
634
1712
  median_ms: median(durations),
635
1713
  p90_ms: percentile(durations, 0.9),
636
1714
  total_sessions: allSessions.length,
1715
+ source,
1716
+ stampedDivergence,
637
1717
  });
638
1718
  }
639
1719
  if (requested.has('eventsPerSession')) {
@@ -644,6 +1724,8 @@ function sessionMetrics(events, { event, metrics = ['count', 'duration', 'events
644
1724
  median: median(eventCounts),
645
1725
  p90: percentile(eventCounts, 0.9),
646
1726
  total_sessions: allSessions.length,
1727
+ source,
1728
+ stampedDivergence,
647
1729
  });
648
1730
  }
649
1731
  return out;