@ak--47/dungeon-master 1.4.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +158 -0
- package/.claude/skills/create-dungeon/SKILL.md +464 -0
- package/.claude/skills/verify-dungeon/SKILL.md +157 -0
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
- package/.claude/skills/write-hooks/SKILL.md +468 -0
- package/CHANGELOG.md +139 -0
- package/HOOKS.md +1243 -597
- package/README.md +140 -5
- package/dungeons/technical/ad-spend.js +1 -1
- package/dungeons/technical/anonymous-users.js +1 -1
- package/dungeons/technical/array-of-object-lookup.js +1 -1
- package/dungeons/technical/datagen-v15-verify.js +74 -0
- package/dungeons/technical/experiments.js +1 -1
- package/dungeons/technical/foobar.js +1 -1
- package/dungeons/technical/group-analytics.js +1 -1
- package/dungeons/technical/mirror-strategies.js +1 -1
- package/dungeons/technical/nested-objects.js +1 -1
- package/dungeons/technical/retention-cadence.js +1 -1
- package/dungeons/technical/sanity.js +1 -1
- package/dungeons/technical/scale-test.js +1 -1
- package/dungeons/technical/scd.js +1 -1
- package/dungeons/technical/simple.js +1 -1
- package/dungeons/technical/simplest.js +74 -20
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +4 -0
- package/dungeons/vertical/community.js +9 -3
- package/dungeons/vertical/crypto.js +5 -0
- package/dungeons/vertical/dating.js +23 -10
- package/dungeons/vertical/devtools.js +10 -0
- package/dungeons/vertical/ecommerce.js +6 -0
- package/dungeons/vertical/education.js +11 -0
- package/dungeons/vertical/fintech.js +13 -0
- package/dungeons/vertical/fitness.js +10 -0
- package/dungeons/vertical/food-delivery.js +9 -0
- package/dungeons/vertical/gaming.js +10 -0
- package/dungeons/vertical/healthcare.js +5 -0
- package/dungeons/vertical/insurance-application.js +10 -0
- package/dungeons/vertical/logistics.js +8 -1
- package/dungeons/vertical/marketplace.js +7 -0
- package/dungeons/vertical/media.js +8 -0
- package/dungeons/vertical/real-estate.js +7 -1
- package/dungeons/vertical/sass.js +12 -0
- package/dungeons/vertical/social.js +9 -0
- package/dungeons/vertical/travel.js +5 -0
- package/index.js +19 -4
- package/lib/core/config-validator.js +270 -7
- package/lib/core/dungeon-loader.js +2 -5
- package/lib/generators/events.js +12 -13
- package/lib/generators/funnels.js +72 -1
- package/lib/hook-helpers/index.js +1 -0
- package/lib/hook-helpers/inject.js +95 -0
- package/lib/orchestrators/user-loop.js +478 -29
- package/lib/templates/macro-presets.js +39 -9
- package/lib/utils/utils.js +16 -79
- package/lib/verify/counting.js +320 -0
- package/lib/verify/emulate-breakdown.js +512 -108
- package/lib/verify/funnel-engine.js +539 -0
- package/lib/verify/identity.js +78 -0
- package/lib/verify/index.js +19 -0
- package/lib/verify/verify-dungeon.js +58 -0
- package/package.json +4 -2
- package/types.d.ts +237 -4
- package/scripts/smoke-test-all.mjs +0 -162
|
@@ -23,55 +23,85 @@
|
|
|
23
23
|
* across [FIXED_BEGIN - 30d, FIXED_BEGIN] so they don't all stack at day 0).
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Tuned tail_ratio targets (foobar 89-day, post-engine-bunchiness fix):
|
|
28
|
+
*
|
|
29
|
+
* flat: ~1.1 (≈ 1.0 baseline + small cumulative-acquisition uptick)
|
|
30
|
+
* steady: ~1.1 (slightly higher uptick from 0.1 bias)
|
|
31
|
+
* growth: ~1.4 (clear visible uptrend)
|
|
32
|
+
* viral: ~2.2 (hockey stick — clearly distinguishable)
|
|
33
|
+
* decline: ~1.0 (flat — pure-dungeon limitation, see decline JSDoc)
|
|
34
|
+
*
|
|
35
|
+
* All five remain DIRECTIONALLY distinguishable on aggregate event count over
|
|
36
|
+
* time. Magnitude diverges from `research/end-bunchiness.md` v1.3 targets
|
|
37
|
+
* (which were measured pre-engine-bunchiness-fix); the new targets are what's
|
|
38
|
+
* achievable with the v1.5 engine fix's per-cluster trapezoidal shape.
|
|
39
|
+
*/
|
|
40
|
+
|
|
26
41
|
/** @type {Record<string, {bornRecentBias: number, percentUsersBornInDataset: number, preExistingSpread: 'pinned'|'uniform'}>} */
|
|
27
42
|
export const MACRO_PRESETS = {
|
|
28
43
|
/**
|
|
29
44
|
* flat (DEFAULT) — Mature product, no growth narrative.
|
|
30
|
-
*
|
|
45
|
+
* Foobar tail_ratio ≈ 1.1 (very slight uptrend from 12% acquisition).
|
|
46
|
+
* Pure weekly oscillation dominates the visual.
|
|
31
47
|
*/
|
|
32
48
|
flat: {
|
|
33
49
|
bornRecentBias: 0,
|
|
34
|
-
percentUsersBornInDataset:
|
|
50
|
+
percentUsersBornInDataset: 12,
|
|
35
51
|
preExistingSpread: 'uniform',
|
|
36
52
|
},
|
|
37
53
|
|
|
38
54
|
/**
|
|
39
55
|
* steady — Lightly-growing SaaS.
|
|
40
|
-
* Slight uptrend without any visible spike
|
|
56
|
+
* Foobar tail_ratio ≈ 1.1. Slight uptrend without any visible spike.
|
|
57
|
+
* Effectively similar to "flat" on a flat dungeon — the slight bornRecentBias
|
|
58
|
+
* shows up more clearly when paired with hooks that magnify late-cohort
|
|
59
|
+
* activity (e.g., engagement decay scoped to early users).
|
|
41
60
|
*/
|
|
42
61
|
steady: {
|
|
43
62
|
bornRecentBias: 0.1,
|
|
44
|
-
percentUsersBornInDataset:
|
|
63
|
+
percentUsersBornInDataset: 12,
|
|
45
64
|
preExistingSpread: 'uniform',
|
|
46
65
|
},
|
|
47
66
|
|
|
48
67
|
/**
|
|
49
68
|
* growth — Visible uptrend story without the meteoric blow-up.
|
|
69
|
+
* Foobar tail_ratio ≈ 1.4. Clearly distinguishable from flat/steady.
|
|
50
70
|
* Use when the dataset wants to show clear acquisition over time.
|
|
51
71
|
*/
|
|
52
72
|
growth: {
|
|
53
73
|
bornRecentBias: 0.3,
|
|
54
|
-
percentUsersBornInDataset:
|
|
74
|
+
percentUsersBornInDataset: 30,
|
|
55
75
|
preExistingSpread: 'pinned',
|
|
56
76
|
},
|
|
57
77
|
|
|
58
78
|
/**
|
|
59
79
|
* viral — Hockey-stick acquisition.
|
|
60
|
-
* Strong late-window ramp.
|
|
80
|
+
* Foobar tail_ratio ≈ 2.2. Strong late-window ramp.
|
|
81
|
+
* Pair with persona/feature hooks for the full effect.
|
|
61
82
|
*/
|
|
62
83
|
viral: {
|
|
63
84
|
bornRecentBias: 0.6,
|
|
64
|
-
percentUsersBornInDataset:
|
|
85
|
+
percentUsersBornInDataset: 55,
|
|
65
86
|
preExistingSpread: 'pinned',
|
|
66
87
|
},
|
|
67
88
|
|
|
68
89
|
/**
|
|
69
90
|
* decline — Sunsetting product, churning users.
|
|
70
|
-
*
|
|
91
|
+
* Foobar tail_ratio ≈ 1.0 — **the bornRecentBias mechanism alone CANNOT
|
|
92
|
+
* produce a downtrend** without a churn mechanism. Born-early users
|
|
93
|
+
* generate events throughout `[birth, FIXED_NOW]`, so they contribute to
|
|
94
|
+
* the right edge as much as the left. To get a real visible downtrend,
|
|
95
|
+
* pair this preset with `engagementDecay` or a hook that drops late
|
|
96
|
+
* events for early-cohort users (see HOOKS.md). On a pure dungeon
|
|
97
|
+
* (foobar) this preset is effectively flat with a slight early-cohort
|
|
98
|
+
* skew. v1.3-era pre-engine-fix verticals (ai-platform/sass/dating)
|
|
99
|
+
* measured tail_ratio ≈ 0.80 for decline because their hooks include
|
|
100
|
+
* implicit churn behavior.
|
|
71
101
|
*/
|
|
72
102
|
decline: {
|
|
73
103
|
bornRecentBias: -0.3,
|
|
74
|
-
percentUsersBornInDataset:
|
|
104
|
+
percentUsersBornInDataset: 5,
|
|
75
105
|
preExistingSpread: 'uniform',
|
|
76
106
|
},
|
|
77
107
|
};
|
package/lib/utils/utils.js
CHANGED
|
@@ -188,7 +188,6 @@ function datesBetween(start, end) {
|
|
|
188
188
|
* @param {any} end
|
|
189
189
|
*/
|
|
190
190
|
function day(start, end) {
|
|
191
|
-
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
192
191
|
if (!start) start = DATASET_NOW.subtract(30, 'd').toISOString();
|
|
193
192
|
if (!end) end = DATASET_NOW.toISOString();
|
|
194
193
|
const chance = getChance();
|
|
@@ -1063,8 +1062,10 @@ function validateEventConfig(events) {
|
|
|
1063
1062
|
}
|
|
1064
1063
|
|
|
1065
1064
|
function validTime(chosenTime, earliestTime, latestTime) {
|
|
1066
|
-
|
|
1067
|
-
|
|
1065
|
+
// Fallback: use module-scoped DATASET_BEGIN/NOW (set by orchestrator via
|
|
1066
|
+
// setDatasetBegin/setDatasetNow). v1.5: no longer reads `global.FIXED_*`.
|
|
1067
|
+
if (!earliestTime) earliestTime = DATASET_BEGIN.unix();
|
|
1068
|
+
if (!latestTime) latestTime = DATASET_NOW.unix();
|
|
1068
1069
|
|
|
1069
1070
|
if (typeof chosenTime === 'number') {
|
|
1070
1071
|
if (chosenTime > 0) {
|
|
@@ -1286,8 +1287,10 @@ const DEFAULT_HOD_WEIGHTS = [
|
|
|
1286
1287
|
];
|
|
1287
1288
|
|
|
1288
1289
|
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS) {
|
|
1289
|
-
|
|
1290
|
-
|
|
1290
|
+
// Fallback: use module-scoped DATASET_BEGIN/NOW (set by orchestrator via
|
|
1291
|
+
// setDatasetBegin/setDatasetNow). v1.5: no longer reads `global.FIXED_*`.
|
|
1292
|
+
if (!earliestTime) earliestTime = DATASET_BEGIN.unix();
|
|
1293
|
+
if (!latestTime) latestTime = DATASET_NOW.unix();
|
|
1291
1294
|
const chance = getChance();
|
|
1292
1295
|
let totalRange = latestTime - earliestTime;
|
|
1293
1296
|
if (totalRange <= 0 || earliestTime > latestTime) {
|
|
@@ -1571,78 +1574,6 @@ function generateSessionId(seedStr) {
|
|
|
1571
1574
|
return [seg(), seg(), seg(), seg()].join("-");
|
|
1572
1575
|
}
|
|
1573
1576
|
|
|
1574
|
-
/**
|
|
1575
|
-
* Redistributes events into temporal clusters (sessions).
|
|
1576
|
-
*
|
|
1577
|
-
* Algorithm:
|
|
1578
|
-
* 1. Sort events by time
|
|
1579
|
-
* 2. Determine number of sessions (total events / avg events per session)
|
|
1580
|
-
* 3. Generate session anchor times using TimeSoup
|
|
1581
|
-
* 4. Assign events round-robin to sessions
|
|
1582
|
-
* 5. Within each session, retime events with tight spacing (5-300s apart)
|
|
1583
|
-
* 6. Regenerate insert_ids for retimed events
|
|
1584
|
-
* 7. Re-sort by time
|
|
1585
|
-
*
|
|
1586
|
-
* Mutates events in place. Does NOT assign session_id (call assignSessionIds after).
|
|
1587
|
-
*
|
|
1588
|
-
* @param {Object[]} events - Array of event objects with .time (ISO string)
|
|
1589
|
-
* @param {number} timeoutMinutes - Session timeout in minutes (used to determine intra-session spacing)
|
|
1590
|
-
* @param {Object} soupParams - Parameters for TimeSoup anchor generation
|
|
1591
|
-
*/
|
|
1592
|
-
function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
1593
|
-
if (events.length < 2) return;
|
|
1594
|
-
|
|
1595
|
-
const chance = getChance();
|
|
1596
|
-
const { earliestTime, latestTime, peaks, deviation, mean,
|
|
1597
|
-
dayOfWeekWeights, hourOfDayWeights } = soupParams;
|
|
1598
|
-
|
|
1599
|
-
// Sort by time first
|
|
1600
|
-
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
1601
|
-
|
|
1602
|
-
// Determine number of sessions: target 3-8 events per session
|
|
1603
|
-
const eventsPerSession = chance.integer({ min: 3, max: 8 });
|
|
1604
|
-
const numSessions = Math.max(1, Math.ceil(events.length / eventsPerSession));
|
|
1605
|
-
|
|
1606
|
-
// Generate session anchor times using TimeSoup
|
|
1607
|
-
const anchors = [];
|
|
1608
|
-
for (let i = 0; i < numSessions; i++) {
|
|
1609
|
-
const soupTime = TimeSoup(earliestTime, latestTime, peaks, deviation, mean,
|
|
1610
|
-
dayOfWeekWeights, hourOfDayWeights);
|
|
1611
|
-
anchors.push(soupTime);
|
|
1612
|
-
}
|
|
1613
|
-
anchors.sort((a, b) => a - b);
|
|
1614
|
-
|
|
1615
|
-
// Distribute events across sessions round-robin (preserving original order → temporal order)
|
|
1616
|
-
const sessionBuckets = anchors.map(() => []);
|
|
1617
|
-
for (let i = 0; i < events.length; i++) {
|
|
1618
|
-
const bucketIndex = Math.min(i % numSessions, numSessions - 1);
|
|
1619
|
-
sessionBuckets[bucketIndex].push(events[i]);
|
|
1620
|
-
}
|
|
1621
|
-
|
|
1622
|
-
// Retime events within each session
|
|
1623
|
-
for (let s = 0; s < numSessions; s++) {
|
|
1624
|
-
const bucket = sessionBuckets[s];
|
|
1625
|
-
if (bucket.length === 0) continue;
|
|
1626
|
-
|
|
1627
|
-
let currentTime = anchors[s];
|
|
1628
|
-
for (let e = 0; e < bucket.length; e++) {
|
|
1629
|
-
const ev = bucket[e];
|
|
1630
|
-
const clampedTime = Math.min(currentTime, latestTime);
|
|
1631
|
-
|
|
1632
|
-
ev.time = dayjs.unix(clampedTime).toISOString();
|
|
1633
|
-
// Regenerate insert_id to match new time
|
|
1634
|
-
const distinctId = ev.user_id || ev.device_id || ev.distinct_id || '';
|
|
1635
|
-
ev.insert_id = quickHash(`${ev.event}-${ev.time}-${distinctId}`);
|
|
1636
|
-
|
|
1637
|
-
// Advance time within session: 5-300 seconds (5s to 5min)
|
|
1638
|
-
currentTime += chance.integer({ min: 5, max: 300 });
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
|
|
1642
|
-
// Re-sort by time
|
|
1643
|
-
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
1577
|
/**
|
|
1647
1578
|
* Assigns session IDs to a chronologically sorted array of events.
|
|
1648
1579
|
* A new session starts when:
|
|
@@ -1661,6 +1592,7 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1661
1592
|
|
|
1662
1593
|
const timeoutMs = timeoutMinutes * 60 * 1000;
|
|
1663
1594
|
const maxSessionMs = 24 * 60 * 60 * 1000;
|
|
1595
|
+
const DAY_MS = 86400 * 1000;
|
|
1664
1596
|
|
|
1665
1597
|
// Derive a stable seed from the first event so session IDs are deterministic
|
|
1666
1598
|
// across runs without consuming the main seeded RNG stream.
|
|
@@ -1668,15 +1600,21 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1668
1600
|
let currentSessionId = generateSessionId(`${userKey}-${events[0].time}`);
|
|
1669
1601
|
let sessionStartMs = new Date(events[0].time).getTime();
|
|
1670
1602
|
let lastEventMs = sessionStartMs;
|
|
1603
|
+
let sessionDayIdx = Math.floor(sessionStartMs / DAY_MS);
|
|
1671
1604
|
|
|
1672
1605
|
for (const event of events) {
|
|
1673
1606
|
const eventMs = new Date(event.time).getTime();
|
|
1674
1607
|
const gapFromLast = eventMs - lastEventMs;
|
|
1675
1608
|
const sessionDuration = eventMs - sessionStartMs;
|
|
1609
|
+
const dayIdx = Math.floor(eventMs / DAY_MS);
|
|
1676
1610
|
|
|
1677
|
-
|
|
1611
|
+
// Mixpanel session_query.cpp:828-830, 911 — sessions terminate on a
|
|
1612
|
+
// day-index change in qtz (UTC here). Three reset triggers: timeout
|
|
1613
|
+
// gap, max-session cap, OR day boundary crossed.
|
|
1614
|
+
if (gapFromLast > timeoutMs || sessionDuration > maxSessionMs || dayIdx !== sessionDayIdx) {
|
|
1678
1615
|
currentSessionId = generateSessionId(`${userKey}-${event.time}`);
|
|
1679
1616
|
sessionStartMs = eventMs;
|
|
1617
|
+
sessionDayIdx = dayIdx;
|
|
1680
1618
|
}
|
|
1681
1619
|
|
|
1682
1620
|
event.session_id = currentSessionId;
|
|
@@ -1747,7 +1685,6 @@ export {
|
|
|
1747
1685
|
formatDuration,
|
|
1748
1686
|
generateSessionId,
|
|
1749
1687
|
assignSessionIds,
|
|
1750
|
-
bunchIntoSessions,
|
|
1751
1688
|
setDatasetNow,
|
|
1752
1689
|
setDatasetBegin,
|
|
1753
1690
|
dateRange,
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Counting and aggregation helpers matching Mixpanel's analytics semantics.
|
|
3
|
+
*
|
|
4
|
+
* These primitives differ from naive SQL in important ways. Each helper
|
|
5
|
+
* documents the Mixpanel source file that defines the rule and the
|
|
6
|
+
* specific divergence from `COUNT(*)` / `AVG(x)` / etc.
|
|
7
|
+
*
|
|
8
|
+
* References (from `mixpanel/analytics`):
|
|
9
|
+
* - `backend/arb/reader/queries/addiction_query.cpp` — distinct-period counting
|
|
10
|
+
* - `backend/arb/reader/queries/normal_query.cpp` — null-aware AVG/SUM/MIN/MAX
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { toMs } from '../hook-helpers/_internal.js';
|
|
14
|
+
|
|
15
|
+
const SECONDS_PER_UNIT = {
|
|
16
|
+
hour: 3600,
|
|
17
|
+
day: 86400,
|
|
18
|
+
week: 7 * 86400,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Count distinct time periods on which a user fired a given event.
|
|
23
|
+
*
|
|
24
|
+
* Two related rules exist in Mixpanel:
|
|
25
|
+
*
|
|
26
|
+
* 1. **Calendar bucket** (default here, `algorithm: 'calendar'`):
|
|
27
|
+
* `COUNT(DISTINCT date_trunc(unit, time))` in UTC. This is what the
|
|
28
|
+
* Mixpanel UI presents — frequency distribution charts bucket events
|
|
29
|
+
* into calendar hours/days/weeks.
|
|
30
|
+
*
|
|
31
|
+
* 2. **Rolling window** (`algorithm: 'rolling'`): the addiction_query.cpp
|
|
32
|
+
* rule, `qtz_time >= last_counted + seconds_for_unit(unit)`. This is
|
|
33
|
+
* Mixpanel's internal C++ implementation. It diverges from calendar
|
|
34
|
+
* bucketing at unit boundaries — events at 23:59 and 00:01 next day
|
|
35
|
+
* register as 1 rolling-window period (gap 120s < 86400s) but 2
|
|
36
|
+
* calendar-day periods.
|
|
37
|
+
*
|
|
38
|
+
* The default is `calendar` because:
|
|
39
|
+
* - It matches what users actually see in Mixpanel reports.
|
|
40
|
+
* - It aligns with `injectOnNewDays`, which classifies days by
|
|
41
|
+
* `Math.floor(t / DAY_MS)` to find empty days. Mixing the two
|
|
42
|
+
* algorithms causes the atom and verifier to disagree at boundaries.
|
|
43
|
+
*
|
|
44
|
+
* Use `algorithm: 'rolling'` only when you're verifying behavior that
|
|
45
|
+
* specifically depends on the C++ rolling-window check.
|
|
46
|
+
*
|
|
47
|
+
* Reference: `mixpanel/analytics`
|
|
48
|
+
* - calendar bucketing: implicit in the UI / Insights reports
|
|
49
|
+
* - rolling-window: `backend/arb/reader/queries/addiction_query.cpp`
|
|
50
|
+
*
|
|
51
|
+
* @param {Object[]} events - Events to scan (mixed types OK).
|
|
52
|
+
* @param {string} eventName - Event name to filter for.
|
|
53
|
+
* @param {('hour'|'day'|'week')} [unit='day']
|
|
54
|
+
* @param {Object} [options]
|
|
55
|
+
* @param {('calendar'|'rolling')} [options.algorithm='calendar']
|
|
56
|
+
* @returns {number} Distinct period count.
|
|
57
|
+
*/
|
|
58
|
+
export function countDistinctPeriods(events, eventName, unit = 'day', options = {}) {
|
|
59
|
+
const seconds = SECONDS_PER_UNIT[unit];
|
|
60
|
+
if (!seconds) throw new Error(`countDistinctPeriods: unsupported unit "${unit}"`);
|
|
61
|
+
if (!Array.isArray(events) || !events.length) return 0;
|
|
62
|
+
const matches = events
|
|
63
|
+
.filter(e => e && e.event === eventName)
|
|
64
|
+
.map(e => toMs(e.time))
|
|
65
|
+
.filter(ms => Number.isFinite(ms));
|
|
66
|
+
if (!matches.length) return 0;
|
|
67
|
+
const unitMs = seconds * 1000;
|
|
68
|
+
const { algorithm = 'calendar' } = options;
|
|
69
|
+
|
|
70
|
+
if (algorithm === 'calendar') {
|
|
71
|
+
// Calendar bucket — UTC floor by unit. Matches what Mixpanel's UI
|
|
72
|
+
// shows and what `injectOnNewDays` uses internally.
|
|
73
|
+
const buckets = new Set();
|
|
74
|
+
for (const t of matches) buckets.add(Math.floor(t / unitMs));
|
|
75
|
+
return buckets.size;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Rolling window — addiction_query.cpp semantics.
|
|
79
|
+
matches.sort((a, b) => a - b);
|
|
80
|
+
let count = 0;
|
|
81
|
+
let lastCountedMs = -Infinity;
|
|
82
|
+
for (const t of matches) {
|
|
83
|
+
if (t >= lastCountedMs + unitMs) {
|
|
84
|
+
count++;
|
|
85
|
+
lastCountedMs = t;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return count;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Null-aware average matching Mixpanel's aggregation semantics.
|
|
93
|
+
*
|
|
94
|
+
* Reference: `backend/arb/reader/queries/normal_query.cpp` ACTION_TYPE_AVERAGE:
|
|
95
|
+
*
|
|
96
|
+
* if (action_value.type == VALUE_TYPE_NUMBER && !std::isnan(value)) {
|
|
97
|
+
* v->average.sum += number;
|
|
98
|
+
* v->average.count += upsampling_factor;
|
|
99
|
+
* }
|
|
100
|
+
*
|
|
101
|
+
* Skips null, undefined, NaN, and non-numeric values from BOTH numerator
|
|
102
|
+
* and denominator. Returns null when no numeric values exist.
|
|
103
|
+
*
|
|
104
|
+
* Differs from naive `SUM(x) / COUNT(*)` which inflates the denominator
|
|
105
|
+
* by counting rows where x is missing — diluting the average toward 0.
|
|
106
|
+
*
|
|
107
|
+
* @param {*[]} values
|
|
108
|
+
* @returns {number|null}
|
|
109
|
+
*/
|
|
110
|
+
export function nullAwareAvg(values) {
|
|
111
|
+
if (!Array.isArray(values) || !values.length) return null;
|
|
112
|
+
let sum = 0;
|
|
113
|
+
let count = 0;
|
|
114
|
+
for (const v of values) {
|
|
115
|
+
if (typeof v === 'number' && Number.isFinite(v)) {
|
|
116
|
+
sum += v;
|
|
117
|
+
count++;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return count ? sum / count : null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Null-aware sum. Skips null/undefined/NaN/non-numeric silently.
|
|
125
|
+
*
|
|
126
|
+
* Reference: `normal_query.cpp` ACTION_TYPE_SUM — same numeric guard as AVG.
|
|
127
|
+
* Differs from naive SQL SUM only when missing values are coalesced to 0
|
|
128
|
+
* upstream; in JS arrays missing values are typically `undefined` which
|
|
129
|
+
* produces NaN under `+`.
|
|
130
|
+
*
|
|
131
|
+
* @param {*[]} values
|
|
132
|
+
* @returns {number}
|
|
133
|
+
*/
|
|
134
|
+
export function nullAwareSum(values) {
|
|
135
|
+
if (!Array.isArray(values) || !values.length) return 0;
|
|
136
|
+
let sum = 0;
|
|
137
|
+
for (const v of values) {
|
|
138
|
+
if (typeof v === 'number' && Number.isFinite(v)) sum += v;
|
|
139
|
+
}
|
|
140
|
+
return sum;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Null-aware min/max. Returns null when no numeric values exist.
|
|
145
|
+
*
|
|
146
|
+
* Reference: `normal_query.cpp` ACTION_TYPE_EXTREMES — only records numeric
|
|
147
|
+
* values. Mixpanel starts max at -Infinity and min at +Infinity; we return
|
|
148
|
+
* null instead of those sentinels when no values were recorded.
|
|
149
|
+
*
|
|
150
|
+
* @param {*[]} values
|
|
151
|
+
* @param {('min'|'max')} mode
|
|
152
|
+
* @returns {number|null}
|
|
153
|
+
*/
|
|
154
|
+
export function nullAwareExtreme(values, mode) {
|
|
155
|
+
if (!Array.isArray(values) || !values.length) return null;
|
|
156
|
+
let extreme = mode === 'min' ? Infinity : -Infinity;
|
|
157
|
+
let any = false;
|
|
158
|
+
for (const v of values) {
|
|
159
|
+
if (typeof v === 'number' && Number.isFinite(v)) {
|
|
160
|
+
any = true;
|
|
161
|
+
if (mode === 'min') {
|
|
162
|
+
if (v < extreme) extreme = v;
|
|
163
|
+
} else {
|
|
164
|
+
if (v > extreme) extreme = v;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return any ? extreme : null;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Partition events into time buckets by UTC calendar (`day`, `week`, or
|
|
173
|
+
* `month`). Used by `emulateBreakdown` when `timeBucket` is set to slice any
|
|
174
|
+
* breakdown into a trend over time.
|
|
175
|
+
*
|
|
176
|
+
* Period labels:
|
|
177
|
+
* - `'day'` → `YYYY-MM-DD`
|
|
178
|
+
* - `'week'` → ISO week `YYYY-Www` (Monday-anchored)
|
|
179
|
+
* - `'month'` → `YYYY-MM`
|
|
180
|
+
*
|
|
181
|
+
* Mixpanel parity: bucket boundaries are computed in UTC. Production Mixpanel
|
|
182
|
+
* uses query timezone (qtz); pass timestamps already shifted to qtz if you
|
|
183
|
+
* need that behavior. ISO week is Monday-anchored
|
|
184
|
+
* (matches `eval_node.c:3641-3643`).
|
|
185
|
+
*
|
|
186
|
+
* Empty-bucket backfill (Mixpanel `normal_query.cpp:352-356, 310-313` emits
|
|
187
|
+
* zero rows for empty intervals): when `options.from` AND `options.to` are
|
|
188
|
+
* provided, the result enumerates every bucket in `[from, to]` and emits
|
|
189
|
+
* `{ period, events: [] }` for buckets with no events. Without `from`/`to`,
|
|
190
|
+
* only buckets that contain at least one event are returned.
|
|
191
|
+
*
|
|
192
|
+
* @param {Object[]} events
|
|
193
|
+
* @param {('day'|'week'|'month')} bucket
|
|
194
|
+
* @param {Object} [options]
|
|
195
|
+
* @param {number|string} [options.from] - Inclusive range start (ms or ISO).
|
|
196
|
+
* @param {number|string} [options.to] - Inclusive range end (ms or ISO).
|
|
197
|
+
* @returns {Array<{ period: string, events: Object[] }>}
|
|
198
|
+
*/
|
|
199
|
+
export function partitionByTimeBucket(events, bucket, options = {}) {
|
|
200
|
+
const groups = new Map();
|
|
201
|
+
if (Array.isArray(events)) {
|
|
202
|
+
for (const ev of events) {
|
|
203
|
+
const ms = toMs(ev && ev.time);
|
|
204
|
+
if (!Number.isFinite(ms)) continue;
|
|
205
|
+
const period = formatBucket(ms, bucket);
|
|
206
|
+
if (!groups.has(period)) groups.set(period, []);
|
|
207
|
+
groups.get(period).push(ev);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const fromMs = options.from != null ? toMs(options.from) : null;
|
|
211
|
+
const toMsBound = options.to != null ? toMs(options.to) : null;
|
|
212
|
+
if (Number.isFinite(fromMs) && Number.isFinite(toMsBound)) {
|
|
213
|
+
// Enumerate every bucket period in [from, to] and seed empties.
|
|
214
|
+
for (const period of enumerateBucketPeriods(fromMs, toMsBound, bucket)) {
|
|
215
|
+
if (!groups.has(period)) groups.set(period, []);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return [...groups.entries()]
|
|
219
|
+
.map(([period, evs]) => ({ period, events: evs }))
|
|
220
|
+
.sort((a, b) => a.period.localeCompare(b.period));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Enumerate canonical bucket period labels covering `[fromMs, toMs]` UTC.
|
|
225
|
+
* Used to backfill empty rows when the caller supplies a trend axis.
|
|
226
|
+
*
|
|
227
|
+
* @param {number} fromMs
|
|
228
|
+
* @param {number} toMs
|
|
229
|
+
* @param {('day'|'week'|'month')} bucket
|
|
230
|
+
* @returns {string[]}
|
|
231
|
+
*/
|
|
232
|
+
function enumerateBucketPeriods(fromMs, toMs, bucket) {
|
|
233
|
+
const out = [];
|
|
234
|
+
if (!(toMs >= fromMs)) return out;
|
|
235
|
+
if (bucket === 'day') {
|
|
236
|
+
const start = Math.floor(fromMs / 86400_000);
|
|
237
|
+
const end = Math.floor(toMs / 86400_000);
|
|
238
|
+
for (let d = start; d <= end; d++) out.push(formatBucket(d * 86400_000, 'day'));
|
|
239
|
+
} else if (bucket === 'week') {
|
|
240
|
+
// Walk by 7 days starting from fromMs; rely on label dedup via Set.
|
|
241
|
+
const seen = new Set();
|
|
242
|
+
for (let t = fromMs; t <= toMs; t += 7 * 86400_000) {
|
|
243
|
+
const p = formatBucket(t, 'week'); if (!seen.has(p)) { seen.add(p); out.push(p); }
|
|
244
|
+
}
|
|
245
|
+
const last = formatBucket(toMs, 'week');
|
|
246
|
+
if (!seen.has(last)) out.push(last);
|
|
247
|
+
} else if (bucket === 'month') {
|
|
248
|
+
const seen = new Set();
|
|
249
|
+
const start = new Date(fromMs);
|
|
250
|
+
const end = new Date(toMs);
|
|
251
|
+
let y = start.getUTCFullYear(), m = start.getUTCMonth();
|
|
252
|
+
const yEnd = end.getUTCFullYear(), mEnd = end.getUTCMonth();
|
|
253
|
+
while (y < yEnd || (y === yEnd && m <= mEnd)) {
|
|
254
|
+
const p = formatBucket(Date.UTC(y, m, 1), 'month');
|
|
255
|
+
if (!seen.has(p)) { seen.add(p); out.push(p); }
|
|
256
|
+
m++; if (m > 11) { m = 0; y++; }
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return out;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function pad2(n) { return n < 10 ? `0${n}` : `${n}`; }
|
|
263
|
+
|
|
264
|
+
function formatBucket(ms, bucket) {
|
|
265
|
+
const d = new Date(ms);
|
|
266
|
+
if (bucket === 'day') {
|
|
267
|
+
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}`;
|
|
268
|
+
}
|
|
269
|
+
if (bucket === 'month') {
|
|
270
|
+
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}`;
|
|
271
|
+
}
|
|
272
|
+
if (bucket === 'week') {
|
|
273
|
+
// ISO week: Mon-anchored. Algorithm: shift to Thursday of the week,
|
|
274
|
+
// take year of that Thursday + week number relative to Jan-1 of that
|
|
275
|
+
// year's Monday-of-Thursday.
|
|
276
|
+
const date = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
|
|
277
|
+
const dayNum = (date.getUTCDay() + 6) % 7; // Mon=0..Sun=6
|
|
278
|
+
date.setUTCDate(date.getUTCDate() - dayNum + 3); // Thursday of this week
|
|
279
|
+
const year = date.getUTCFullYear();
|
|
280
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
281
|
+
const jan4DayNum = (jan4.getUTCDay() + 6) % 7;
|
|
282
|
+
const week1Mon = new Date(Date.UTC(year, 0, 4 - jan4DayNum));
|
|
283
|
+
const weekNum = Math.floor((date.getTime() - week1Mon.getTime()) / (7 * 86400_000)) + 1;
|
|
284
|
+
return `${year}-W${pad2(weekNum)}`;
|
|
285
|
+
}
|
|
286
|
+
throw new Error(`partitionByTimeBucket: unsupported bucket "${bucket}"`);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Bin a user by their distinct-period count of an event. Combines
|
|
291
|
+
* `countDistinctPeriods` with bin classification for cohort assignment.
|
|
292
|
+
*
|
|
293
|
+
* Each bin entry is `[min, max]` with `min` inclusive, `max` exclusive.
|
|
294
|
+
* Returns the first matching bin name, or `null` if no bin matches.
|
|
295
|
+
*
|
|
296
|
+
* Uses calendar-bucket counting by default (matches Mixpanel UI). Pass
|
|
297
|
+
* `options.algorithm: 'rolling'` to use the addiction_query.cpp rule
|
|
298
|
+
* instead — see `countDistinctPeriods` for the difference.
|
|
299
|
+
*
|
|
300
|
+
* Replaces total-event counting for any analysis that targets Mixpanel's
|
|
301
|
+
* frequency distribution (which counts distinct periods, not total events).
|
|
302
|
+
*
|
|
303
|
+
* @param {Object[]} events
|
|
304
|
+
* @param {string} eventName
|
|
305
|
+
* @param {Object<string, [number, number]>} bins
|
|
306
|
+
* @param {('hour'|'day'|'week')} [unit='day']
|
|
307
|
+
* @param {Object} [options]
|
|
308
|
+
* @param {('calendar'|'rolling')} [options.algorithm='calendar']
|
|
309
|
+
* @returns {string|null}
|
|
310
|
+
*/
|
|
311
|
+
export function binByDistinctPeriods(events, eventName, bins, unit = 'day', options = {}) {
|
|
312
|
+
const periods = countDistinctPeriods(events, eventName, unit, options);
|
|
313
|
+
if (!bins || typeof bins !== 'object') return null;
|
|
314
|
+
for (const [name, range] of Object.entries(bins)) {
|
|
315
|
+
if (!Array.isArray(range) || range.length !== 2) continue;
|
|
316
|
+
const [min, max] = range;
|
|
317
|
+
if (periods >= min && periods < max) return name;
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|