@ak--47/dungeon-master 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +96 -0
- package/.claude/skills/create-project/provision.mjs +336 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +211 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +3 -2
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared value-coercion + filter-comparison helpers matching Mixpanel's ARB
|
|
3
|
+
* engine semantics. Two DIFFERENT rulebooks live here — do not mix them up:
|
|
4
|
+
*
|
|
5
|
+
* 1. SEGMENT IDENTITY (breakdown bucketing) is case-SENSITIVE and type-tagged.
|
|
6
|
+
* ARB hashes the raw typed value: strings hash their raw bytes with no case
|
|
7
|
+
* folding (hash_value.c:114-115 → hash_string_value_with_seed, raw XXH3
|
|
8
|
+
* over strlen bytes), and segment ordering uses arb_strcmp (cmp.c:24-32).
|
|
9
|
+
* Each value type mixes a distinct tag into the hash (number_tag,
|
|
10
|
+
* string_tag, true/false_tag — hash_value.c:92-97), so the number 1 and
|
|
11
|
+
* the string "1" are DIFFERENT segments even though both display as "1".
|
|
12
|
+
*
|
|
13
|
+
* 2. WHERE-FILTER string comparison is case-INSENSITIVE. value_equal uses
|
|
14
|
+
* arb_strcasecmp (value.c:285), `contains` uses arb_strcaseinstr
|
|
15
|
+
* (eval_node.c:2914), and string relational operators use arb_strcasecmp
|
|
16
|
+
* (eval_node.c:2931). The filter helpers below implement that rulebook and
|
|
17
|
+
* must NEVER be used to bucket breakdown segments.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Coerce a property value to its Mixpanel display/bucket label.
|
|
22
|
+
*
|
|
23
|
+
* - `null` and `undefined` both become the literal string "undefined" — the
|
|
24
|
+
* string-typecast default (`string(prop, "undefined")`) that produces the
|
|
25
|
+
* "undefined" segment in every Insights breakdown (arb_selector.py:889-916).
|
|
26
|
+
* - Booleans → "true"/"false"; numbers → decimal string with -0 normalized to
|
|
27
|
+
* 0 (hash_value.c:111 `v.d = v.d == -0.0 ? 0.0 : v.d`).
|
|
28
|
+
* - Strings pass through UNCHANGED — case-preserving (rulebook 1 above).
|
|
29
|
+
* - Objects JSON-stringify (rare; ARB hashes structurally — hash_object).
|
|
30
|
+
* - Lists are NOT handled here: callers fan out per item (ACTION_TYPE_FOR_EACH,
|
|
31
|
+
* normal_query.cpp:1718-1776) before coercing each item.
|
|
32
|
+
*
|
|
33
|
+
* @param {*} value
|
|
34
|
+
* @returns {string}
|
|
35
|
+
*/
|
|
36
|
+
export function coerceToBreakdownKey(value) {
|
|
37
|
+
if (value === null || value === undefined) return 'undefined';
|
|
38
|
+
if (typeof value === 'boolean') return value ? 'true' : 'false';
|
|
39
|
+
if (typeof value === 'number') return String(Object.is(value, -0) ? 0 : value);
|
|
40
|
+
if (typeof value === 'string') return value;
|
|
41
|
+
return JSON.stringify(value);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Internal segment-identity key: type-tagged so number 1 ≠ string "1"
|
|
46
|
+
* (hash_value.c type tag mixins). Case-preserving for strings.
|
|
47
|
+
*
|
|
48
|
+
* @param {*} value
|
|
49
|
+
* @returns {string}
|
|
50
|
+
*/
|
|
51
|
+
export function breakdownSegmentKey(value) {
|
|
52
|
+
if (value === null || value === undefined) return 'u:';
|
|
53
|
+
if (typeof value === 'boolean') return `b:${value}`;
|
|
54
|
+
if (typeof value === 'number') return `n:${Object.is(value, -0) ? 0 : value}`;
|
|
55
|
+
if (typeof value === 'string') return `s:${value}`;
|
|
56
|
+
return `o:${JSON.stringify(value)}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* WHERE-filter equality. Strings compare case-insensitively
|
|
61
|
+
* (value.c:285 — value_equal → arb_strcasecmp). null == null and
|
|
62
|
+
* undefined == undefined are each equal (value.c VALUE_TYPE_UNDEFINED/NULL
|
|
63
|
+
* cases return equal=true) but null != undefined (type mismatch short-circuits
|
|
64
|
+
* before the switch). Cross-type comparisons are never equal.
|
|
65
|
+
*
|
|
66
|
+
* @param {*} a
|
|
67
|
+
* @param {*} b
|
|
68
|
+
* @returns {boolean}
|
|
69
|
+
*/
|
|
70
|
+
export function filterEquals(a, b) {
|
|
71
|
+
if (typeof a === 'string' && typeof b === 'string') {
|
|
72
|
+
return a.toLowerCase() === b.toLowerCase();
|
|
73
|
+
}
|
|
74
|
+
if (a === null || a === undefined || b === null || b === undefined) {
|
|
75
|
+
return a === b || (a === null && b === null) || (a === undefined && b === undefined);
|
|
76
|
+
}
|
|
77
|
+
return a === b;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* WHERE-filter relational comparison for two strings, case-insensitive
|
|
82
|
+
* (eval_node.c:2931 — string relational operators go through arb_strcasecmp).
|
|
83
|
+
*
|
|
84
|
+
* @param {string} a
|
|
85
|
+
* @param {string} b
|
|
86
|
+
* @returns {number} negative / 0 / positive like strcmp
|
|
87
|
+
*/
|
|
88
|
+
export function filterCompare(a, b) {
|
|
89
|
+
const la = a.toLowerCase(), lb = b.toLowerCase();
|
|
90
|
+
return la < lb ? -1 : la > lb ? 1 : 0;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* WHERE-filter substring test, case-insensitive
|
|
95
|
+
* (eval_node.c:2914 — BINARY_OPERATOR_IN uses arb_strcaseinstr).
|
|
96
|
+
*
|
|
97
|
+
* @param {string} haystack
|
|
98
|
+
* @param {string} needle
|
|
99
|
+
* @returns {boolean}
|
|
100
|
+
*/
|
|
101
|
+
export function filterContains(haystack, needle) {
|
|
102
|
+
return haystack.toLowerCase().includes(needle.toLowerCase());
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* List-valued property membership: ARB's BINARY_OPERATOR_IN with a LIST
|
|
107
|
+
* right operand walks the list and tests per-item value_equal
|
|
108
|
+
* (eval_node.c:2949-2959) — typed equality, case-insensitive for strings
|
|
109
|
+
* (value.c:285). This is the ONLY list operator the query builder emits for
|
|
110
|
+
* list-typed properties ("contains"/"does not contain" compile to in/not-in;
|
|
111
|
+
* LIST_CONTAINS_STRING is never generated by the API layer). Substring
|
|
112
|
+
* matching does NOT apply per item.
|
|
113
|
+
*
|
|
114
|
+
* @param {Array<*>} list
|
|
115
|
+
* @param {*} needle
|
|
116
|
+
* @returns {boolean}
|
|
117
|
+
*/
|
|
118
|
+
function listMembership(list, needle) {
|
|
119
|
+
return list.some(item => filterEquals(item, needle));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Evaluate a `where` filter object against a flat record using the
|
|
124
|
+
* WHERE-filter rulebook (case-insensitive strings). Shape mirrors funnel step
|
|
125
|
+
* filters: `{ prop: value }` (equality) or `{ prop: { op, value } }` with
|
|
126
|
+
* op ∈ eq | neq | gt | lt | gte | lte | contains | not_contains.
|
|
127
|
+
*
|
|
128
|
+
* Relational ops: numbers compare numerically, strings via filterCompare
|
|
129
|
+
* (eval_node.c:2870-2906 numeric branch, :2931 string branch). Mixed-type or
|
|
130
|
+
* null/undefined operands fail relational tests (value_cmp orders by type
|
|
131
|
+
* first — cross-type never satisfies a within-type comparison here).
|
|
132
|
+
*
|
|
133
|
+
* List-valued record properties: eq/contains test per-item membership and
|
|
134
|
+
* neq/not_contains its negation (see listMembership). Raw `==` on
|
|
135
|
+
* list-vs-scalar is a type-mismatch false in ARB, but the query builder never
|
|
136
|
+
* emits it for list props — it emits IN — so the DSL's bare `{ prop: value }`
|
|
137
|
+
* maps to membership rather than silently never matching. Relational ops on
|
|
138
|
+
* list values stay false (scalar-vs-list has no per-item relational in ARB).
|
|
139
|
+
*
|
|
140
|
+
* @param {Object} record flat event/profile record
|
|
141
|
+
* @param {Object<string, *>} [where]
|
|
142
|
+
* @returns {boolean}
|
|
143
|
+
*/
|
|
144
|
+
export function matchesWhere(record, where) {
|
|
145
|
+
if (!where) return true;
|
|
146
|
+
for (const [prop, cond] of Object.entries(where)) {
|
|
147
|
+
const actual = record[prop];
|
|
148
|
+
const { op, value } = (cond && typeof cond === 'object' && !Array.isArray(cond) && 'op' in cond)
|
|
149
|
+
? cond
|
|
150
|
+
: { op: 'eq', value: cond };
|
|
151
|
+
let pass;
|
|
152
|
+
switch (op) {
|
|
153
|
+
case 'eq':
|
|
154
|
+
pass = Array.isArray(actual) ? listMembership(actual, value) : filterEquals(actual, value);
|
|
155
|
+
break;
|
|
156
|
+
case 'neq':
|
|
157
|
+
pass = Array.isArray(actual) ? !listMembership(actual, value) : !filterEquals(actual, value);
|
|
158
|
+
break;
|
|
159
|
+
case 'gt':
|
|
160
|
+
case 'lt':
|
|
161
|
+
case 'gte':
|
|
162
|
+
case 'lte': {
|
|
163
|
+
let cmp;
|
|
164
|
+
if (typeof actual === 'number' && typeof value === 'number') cmp = actual - value;
|
|
165
|
+
else if (typeof actual === 'string' && typeof value === 'string') cmp = filterCompare(actual, value);
|
|
166
|
+
else { pass = false; break; }
|
|
167
|
+
pass = op === 'gt' ? cmp > 0 : op === 'lt' ? cmp < 0 : op === 'gte' ? cmp >= 0 : cmp <= 0;
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
case 'contains':
|
|
171
|
+
pass = Array.isArray(actual)
|
|
172
|
+
? listMembership(actual, value) // membership, NOT per-item substring
|
|
173
|
+
: (typeof actual === 'string' && typeof value === 'string' && filterContains(actual, value));
|
|
174
|
+
break;
|
|
175
|
+
case 'not_contains':
|
|
176
|
+
pass = Array.isArray(actual)
|
|
177
|
+
? !listMembership(actual, value)
|
|
178
|
+
: !(typeof actual === 'string' && typeof value === 'string' && filterContains(actual, value));
|
|
179
|
+
break;
|
|
180
|
+
default:
|
|
181
|
+
throw new Error(`matchesWhere: unknown op "${op}"`);
|
|
182
|
+
}
|
|
183
|
+
if (!pass) return false;
|
|
184
|
+
}
|
|
185
|
+
return true;
|
|
186
|
+
}
|
package/lib/verify/counting.js
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { toMs } from '../hook-helpers/_internal.js';
|
|
14
|
+
import { matchesWhere } from './coerce.js';
|
|
15
|
+
import { buildIdentityMap, resolveUserId } from './identity.js';
|
|
14
16
|
|
|
15
17
|
const SECONDS_PER_UNIT = {
|
|
16
18
|
hour: 3600,
|
|
@@ -21,43 +23,43 @@ const SECONDS_PER_UNIT = {
|
|
|
21
23
|
/**
|
|
22
24
|
* Count distinct time periods on which a user fired a given event.
|
|
23
25
|
*
|
|
24
|
-
* Two related rules exist in Mixpanel
|
|
26
|
+
* Two related rules exist in Mixpanel — v1.6.0 (P1.10, findings #3) names
|
|
27
|
+
* them for what they are:
|
|
25
28
|
*
|
|
26
|
-
* 1.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* into calendar hours/days/weeks.
|
|
29
|
+
* 1. **`'ui-bucket'`** (default): `COUNT(DISTINCT date_trunc(unit, time))`
|
|
30
|
+
* in UTC — calendar-bucket counting, the shape the Mixpanel UI presents
|
|
31
|
+
* in frequency distribution charts.
|
|
30
32
|
*
|
|
31
|
-
* 2.
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
33
|
+
* 2. **`'mixpanel-rolling'`**: the addiction_query.cpp rule,
|
|
34
|
+
* `qtz_time >= interval->last_counted + seconds_for_unit(unit)`
|
|
35
|
+
* (addiction_query_update_history, addiction_query.cpp:363-374). This
|
|
36
|
+
* is what Mixpanel's C++ reader actually computes. It diverges from
|
|
37
|
+
* calendar bucketing at unit boundaries — events at 23:59 and 00:01
|
|
38
|
+
* next day register as 1 rolling period (gap 120s < 86400s) but 2
|
|
39
|
+
* calendar days.
|
|
37
40
|
*
|
|
38
|
-
* The default
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
41
|
+
* **The default does NOT match addiction_query.cpp's rolling rule.** It is
|
|
42
|
+
* kept because it matches what users see in report buckets, and because it
|
|
43
|
+
* aligns with `injectOnNewDays` (which classifies days by
|
|
44
|
+
* `Math.floor(t / DAY_MS)`); mixing the two algorithms makes the atom and
|
|
45
|
+
* verifier disagree at boundaries. For the actual Frequency report output
|
|
46
|
+
* shape (per-interval rolling counters + histogram), use
|
|
47
|
+
* `frequencyHistogram` instead.
|
|
43
48
|
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
* Reference: `mixpanel/analytics`
|
|
48
|
-
* - calendar bucketing: implicit in the UI / Insights reports
|
|
49
|
-
* - rolling-window: `backend/arb/reader/queries/addiction_query.cpp`
|
|
49
|
+
* `'calendar'` and `'rolling'` remain accepted as silent back-compat
|
|
50
|
+
* aliases for `'ui-bucket'` / `'mixpanel-rolling'`.
|
|
50
51
|
*
|
|
51
52
|
* @param {Object[]} events - Events to scan (mixed types OK).
|
|
52
53
|
* @param {string} eventName - Event name to filter for.
|
|
53
54
|
* @param {('hour'|'day'|'week')} [unit='day']
|
|
54
55
|
* @param {Object} [options]
|
|
55
|
-
* @param {('calendar'|'rolling')} [options.algorithm='
|
|
56
|
+
* @param {('ui-bucket'|'mixpanel-rolling'|'calendar'|'rolling')} [options.algorithm='ui-bucket']
|
|
56
57
|
* @returns {number} Distinct period count.
|
|
57
58
|
*/
|
|
58
59
|
export function countDistinctPeriods(events, eventName, unit = 'day', options = {}) {
|
|
59
60
|
const seconds = SECONDS_PER_UNIT[unit];
|
|
60
61
|
if (!seconds) throw new Error(`countDistinctPeriods: unsupported unit "${unit}"`);
|
|
62
|
+
const algorithm = normalizeDistinctPeriodAlgorithm(options.algorithm);
|
|
61
63
|
if (!Array.isArray(events) || !events.length) return 0;
|
|
62
64
|
const matches = events
|
|
63
65
|
.filter(e => e && e.event === eventName)
|
|
@@ -65,9 +67,8 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
|
|
|
65
67
|
.filter(ms => Number.isFinite(ms));
|
|
66
68
|
if (!matches.length) return 0;
|
|
67
69
|
const unitMs = seconds * 1000;
|
|
68
|
-
const { algorithm = 'calendar' } = options;
|
|
69
70
|
|
|
70
|
-
if (algorithm === '
|
|
71
|
+
if (algorithm === 'ui-bucket') {
|
|
71
72
|
// Calendar bucket — UTC floor by unit. Matches what Mixpanel's UI
|
|
72
73
|
// shows and what `injectOnNewDays` uses internally.
|
|
73
74
|
const buckets = new Set();
|
|
@@ -75,7 +76,7 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
|
|
|
75
76
|
return buckets.size;
|
|
76
77
|
}
|
|
77
78
|
|
|
78
|
-
// Rolling window — addiction_query.cpp semantics.
|
|
79
|
+
// Rolling window — addiction_query.cpp:363-374 semantics.
|
|
79
80
|
matches.sort((a, b) => a - b);
|
|
80
81
|
let count = 0;
|
|
81
82
|
let lastCountedMs = -Infinity;
|
|
@@ -88,6 +89,28 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
|
|
|
88
89
|
return count;
|
|
89
90
|
}
|
|
90
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Normalize the countDistinctPeriods algorithm name. `'calendar'` and
|
|
94
|
+
* `'rolling'` are silent v1.5 back-compat aliases; unknown names throw
|
|
95
|
+
* (they previously fell through to the rolling branch silently).
|
|
96
|
+
*
|
|
97
|
+
* @param {string} [algorithm]
|
|
98
|
+
* @returns {('ui-bucket'|'mixpanel-rolling')}
|
|
99
|
+
*/
|
|
100
|
+
function normalizeDistinctPeriodAlgorithm(algorithm) {
|
|
101
|
+
switch (algorithm) {
|
|
102
|
+
case undefined:
|
|
103
|
+
case 'ui-bucket':
|
|
104
|
+
case 'calendar':
|
|
105
|
+
return 'ui-bucket';
|
|
106
|
+
case 'mixpanel-rolling':
|
|
107
|
+
case 'rolling':
|
|
108
|
+
return 'mixpanel-rolling';
|
|
109
|
+
default:
|
|
110
|
+
throw new Error(`countDistinctPeriods: unknown algorithm "${algorithm}" — use 'ui-bucket' or 'mixpanel-rolling'`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
91
114
|
/**
|
|
92
115
|
* Null-aware average matching Mixpanel's aggregation semantics.
|
|
93
116
|
*
|
|
@@ -104,17 +127,40 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
|
|
|
104
127
|
* Differs from naive `SUM(x) / COUNT(*)` which inflates the denominator
|
|
105
128
|
* by counting rows where x is missing — diluting the average toward 0.
|
|
106
129
|
*
|
|
130
|
+
* v1.6.0 (P1.10): `{ flatten: true }` mirrors the ACTION_TYPE_AVERAGE
|
|
131
|
+
* VALUE_TYPE_LIST branch (`normal_query.cpp:1601-1617`): a list-valued
|
|
132
|
+
* property contributes each numeric ITEM independently — every item adds
|
|
133
|
+
* to the numerator AND increments the denominator. One level only
|
|
134
|
+
* (`list_cursor_next` items must themselves be VALUE_TYPE_NUMBER; a
|
|
135
|
+
* nested list item is skipped, not recursed). Non-numeric items inside a
|
|
136
|
+
* list are skipped. Default stays non-flattening: without the opt-in, an
|
|
137
|
+
* array value is non-numeric and is skipped whole — the explicit flag
|
|
138
|
+
* avoids silently changing v1.5 results for 1-item-array data.
|
|
139
|
+
* (ARB's list branch carries no isnan guard — the scalar branch does —
|
|
140
|
+
* but NaN can't arrive via JSON ingestion; we keep the same finite guard
|
|
141
|
+
* on items as on scalars.)
|
|
142
|
+
*
|
|
107
143
|
* @param {*[]} values
|
|
144
|
+
* @param {Object} [options]
|
|
145
|
+
* @param {boolean} [options.flatten=false]
|
|
108
146
|
* @returns {number|null}
|
|
109
147
|
*/
|
|
110
|
-
export function nullAwareAvg(values) {
|
|
148
|
+
export function nullAwareAvg(values, options = {}) {
|
|
111
149
|
if (!Array.isArray(values) || !values.length) return null;
|
|
150
|
+
const { flatten = false } = options;
|
|
112
151
|
let sum = 0;
|
|
113
152
|
let count = 0;
|
|
114
153
|
for (const v of values) {
|
|
115
154
|
if (typeof v === 'number' && Number.isFinite(v)) {
|
|
116
155
|
sum += v;
|
|
117
156
|
count++;
|
|
157
|
+
} else if (flatten && Array.isArray(v)) {
|
|
158
|
+
for (const item of v) {
|
|
159
|
+
if (typeof item === 'number' && Number.isFinite(item)) {
|
|
160
|
+
sum += item;
|
|
161
|
+
count++;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
118
164
|
}
|
|
119
165
|
}
|
|
120
166
|
return count ? sum / count : null;
|
|
@@ -128,14 +174,28 @@ export function nullAwareAvg(values) {
|
|
|
128
174
|
* upstream; in JS arrays missing values are typically `undefined` which
|
|
129
175
|
* produces NaN under `+`.
|
|
130
176
|
*
|
|
177
|
+
* v1.6.0 (P1.10): `{ flatten: true }` mirrors the ACTION_TYPE_SUM
|
|
178
|
+
* VALUE_TYPE_LIST branch (`normal_query.cpp:1585-1600`): each numeric
|
|
179
|
+
* item of a list-valued property adds to the sum independently, one
|
|
180
|
+
* level only, non-numeric items skipped. Same explicit opt-in rationale
|
|
181
|
+
* as `nullAwareAvg`.
|
|
182
|
+
*
|
|
131
183
|
* @param {*[]} values
|
|
184
|
+
* @param {Object} [options]
|
|
185
|
+
* @param {boolean} [options.flatten=false]
|
|
132
186
|
* @returns {number}
|
|
133
187
|
*/
|
|
134
|
-
export function nullAwareSum(values) {
|
|
188
|
+
export function nullAwareSum(values, options = {}) {
|
|
135
189
|
if (!Array.isArray(values) || !values.length) return 0;
|
|
190
|
+
const { flatten = false } = options;
|
|
136
191
|
let sum = 0;
|
|
137
192
|
for (const v of values) {
|
|
138
193
|
if (typeof v === 'number' && Number.isFinite(v)) sum += v;
|
|
194
|
+
else if (flatten && Array.isArray(v)) {
|
|
195
|
+
for (const item of v) {
|
|
196
|
+
if (typeof item === 'number' && Number.isFinite(item)) sum += item;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
139
199
|
}
|
|
140
200
|
return sum;
|
|
141
201
|
}
|
|
@@ -299,6 +359,47 @@ function enumerateBucketPeriods(fromMs, toMs, bucket) {
|
|
|
299
359
|
return out;
|
|
300
360
|
}
|
|
301
361
|
|
|
362
|
+
/**
|
|
363
|
+
* Resolve a bucket period label back to its UTC millisecond bounds.
|
|
364
|
+
* `startMs` inclusive, `endMs` exclusive — the `[start, stop)` convention
|
|
365
|
+
* Mixpanel uses for trend intervals (funnel_query.cpp:1398-1401 anchors
|
|
366
|
+
* step 0 in `[start, stop)`). Inverse of the labels `partitionByTimeBucket`
|
|
367
|
+
* emits: `YYYY-MM-DD` (day), `YYYY-Www` (ISO week, Monday-anchored),
|
|
368
|
+
* `YYYY-MM` (month).
|
|
369
|
+
*
|
|
370
|
+
* @param {string} period
|
|
371
|
+
* @param {('day'|'week'|'month')} bucket
|
|
372
|
+
* @returns {{ startMs: number, endMs: number }}
|
|
373
|
+
*/
|
|
374
|
+
export function bucketBoundsMs(period, bucket) {
|
|
375
|
+
if (bucket === 'day') {
|
|
376
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(period);
|
|
377
|
+
if (!m) throw new Error(`bucketBoundsMs: bad day period "${period}"`);
|
|
378
|
+
const startMs = Date.UTC(+m[1], +m[2] - 1, +m[3]);
|
|
379
|
+
return { startMs, endMs: startMs + 86400_000 };
|
|
380
|
+
}
|
|
381
|
+
if (bucket === 'month') {
|
|
382
|
+
const m = /^(\d{4})-(\d{2})$/.exec(period);
|
|
383
|
+
if (!m) throw new Error(`bucketBoundsMs: bad month period "${period}"`);
|
|
384
|
+
return {
|
|
385
|
+
startMs: Date.UTC(+m[1], +m[2] - 1, 1),
|
|
386
|
+
endMs: Date.UTC(+m[1], +m[2], 1),
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
if (bucket === 'week') {
|
|
390
|
+
const m = /^(\d{4})-W(\d{2})$/.exec(period);
|
|
391
|
+
if (!m) throw new Error(`bucketBoundsMs: bad week period "${period}"`);
|
|
392
|
+
// Monday of ISO week w: week 1 contains Jan 4 (same anchor formatBucket uses).
|
|
393
|
+
const year = +m[1], week = +m[2];
|
|
394
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
395
|
+
const jan4DayNum = (jan4.getUTCDay() + 6) % 7;
|
|
396
|
+
const week1MonMs = Date.UTC(year, 0, 4 - jan4DayNum);
|
|
397
|
+
const startMs = week1MonMs + (week - 1) * 7 * 86400_000;
|
|
398
|
+
return { startMs, endMs: startMs + 7 * 86400_000 };
|
|
399
|
+
}
|
|
400
|
+
throw new Error(`bucketBoundsMs: unsupported bucket "${bucket}"`);
|
|
401
|
+
}
|
|
402
|
+
|
|
302
403
|
function pad2(n) { return n < 10 ? `0${n}` : `${n}`; }
|
|
303
404
|
|
|
304
405
|
function formatBucket(ms, bucket) {
|
|
@@ -333,9 +434,10 @@ function formatBucket(ms, bucket) {
|
|
|
333
434
|
* Each bin entry is `[min, max]` with `min` inclusive, `max` exclusive.
|
|
334
435
|
* Returns the first matching bin name, or `null` if no bin matches.
|
|
335
436
|
*
|
|
336
|
-
* Uses
|
|
337
|
-
*
|
|
338
|
-
*
|
|
437
|
+
* Uses `'ui-bucket'` (calendar) counting by default — matches Mixpanel's
|
|
438
|
+
* UI buckets, NOT addiction_query.cpp's rolling rule. Pass
|
|
439
|
+
* `options.algorithm: 'mixpanel-rolling'` for the C++ rule — see
|
|
440
|
+
* `countDistinctPeriods` for the difference.
|
|
339
441
|
*
|
|
340
442
|
* Replaces total-event counting for any analysis that targets Mixpanel's
|
|
341
443
|
* frequency distribution (which counts distinct periods, not total events).
|
|
@@ -345,7 +447,7 @@ function formatBucket(ms, bucket) {
|
|
|
345
447
|
* @param {Object<string, [number, number]>} bins
|
|
346
448
|
* @param {('hour'|'day'|'week')} [unit='day']
|
|
347
449
|
* @param {Object} [options]
|
|
348
|
-
* @param {('calendar'|'rolling')} [options.algorithm='
|
|
450
|
+
* @param {('ui-bucket'|'mixpanel-rolling'|'calendar'|'rolling')} [options.algorithm='ui-bucket']
|
|
349
451
|
* @returns {string|null}
|
|
350
452
|
*/
|
|
351
453
|
export function binByDistinctPeriods(events, eventName, bins, unit = 'day', options = {}) {
|
|
@@ -358,3 +460,139 @@ export function binByDistinctPeriods(events, eventName, bins, unit = 'day', opti
|
|
|
358
460
|
}
|
|
359
461
|
return null;
|
|
360
462
|
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* The actual Frequency (Addiction) report output shape — per report
|
|
466
|
+
* interval, a histogram of users bucketed by their ROLLING distinct-unit
|
|
467
|
+
* count. This is what `countDistinctPeriods`'s JSDoc points to for real
|
|
468
|
+
* Frequency-report parity.
|
|
469
|
+
*
|
|
470
|
+
* ARB semantics (from `mixpanel/analytics`):
|
|
471
|
+
* - Rolling counter: an event counts iff
|
|
472
|
+
* `qtz_time >= interval->last_counted + seconds_for_unit(unit)`; then
|
|
473
|
+
* `last_counted = qtz_time`, `count++` (`addiction_query_update_history`,
|
|
474
|
+
* `addiction_query.cpp:363-374`). `last_counted` lives per-(user,
|
|
475
|
+
* interval) and is calloc-zeroed — the counter RESETS at every interval
|
|
476
|
+
* boundary, so the first qualifying event in each interval always counts.
|
|
477
|
+
* - Histogram: `intervals_counts[interval][count - 1] += upsampling_factor`
|
|
478
|
+
* for every user history with `count > 0`. Users with ZERO qualifying
|
|
479
|
+
* events in an interval are omitted entirely — there is no zero bucket
|
|
480
|
+
* (`addiction_query_prepare_results` ACTION_TYPE_COUNT,
|
|
481
|
+
* `addiction_query.cpp:546-573`). The print loop emits EVERY interval,
|
|
482
|
+
* zero-filled ones included.
|
|
483
|
+
* - Histogram length: `(interval_seconds + unit_seconds - 1) / unit_seconds`
|
|
484
|
+
* — integer ceil (`addiction_units_per_interval`,
|
|
485
|
+
* `libquery/time/unit.c:108-113`).
|
|
486
|
+
* - Interval selection: forward tiling from range start,
|
|
487
|
+
* `index = (event_time - from_time) / interval_length`
|
|
488
|
+
* (`uniform_intervals_for`, `libquery/time/uniform_intervals.h:48-58`).
|
|
489
|
+
*
|
|
490
|
+
* Range stand-in: ARB takes explicit `--from-date`/`--to-date` query params.
|
|
491
|
+
* This verifier has no query range, so intervals tile forward from UTC
|
|
492
|
+
* midnight of the FIRST event in the stream (any event, not just matches)
|
|
493
|
+
* through the last — the same dataset-boundary discipline `lifecycle` uses.
|
|
494
|
+
* ARB selects the interval by ptz_time and gap-checks by qtz_time; both are
|
|
495
|
+
* raw UTC timestamps here — shift event times upstream for a
|
|
496
|
+
* query-timezone view.
|
|
497
|
+
*
|
|
498
|
+
* @param {Object[]} events
|
|
499
|
+
* @param {Object} [options]
|
|
500
|
+
* @param {string} [options.event] - Event name to count. Required at runtime — throws if missing.
|
|
501
|
+
* @param {('hour'|'day'|'week')} [options.unit='day'] - Rolling unit.
|
|
502
|
+
* @param {number} [options.intervalDays] - Report interval length in days (positive integer). Required at runtime.
|
|
503
|
+
* @param {Object[]} [options.profiles] - Profiles for device→user identity resolution.
|
|
504
|
+
* @returns {Array<{ interval: string, histogram: number[] }>} One row per
|
|
505
|
+
* interval (label = ISO date of the interval start); `histogram[i]` =
|
|
506
|
+
* number of users with rolling count `i + 1` in that interval.
|
|
507
|
+
*/
|
|
508
|
+
export function frequencyHistogram(events, options = {}) {
|
|
509
|
+
const { event, unit = 'day', intervalDays, profiles } = options;
|
|
510
|
+
if (typeof event !== 'string' || !event) {
|
|
511
|
+
throw new Error('frequencyHistogram: event is required');
|
|
512
|
+
}
|
|
513
|
+
const unitSeconds = SECONDS_PER_UNIT[unit];
|
|
514
|
+
if (!unitSeconds) throw new Error(`frequencyHistogram: unsupported unit "${unit}"`);
|
|
515
|
+
if (!Number.isInteger(intervalDays) || intervalDays <= 0) {
|
|
516
|
+
throw new Error('frequencyHistogram: intervalDays must be a positive integer');
|
|
517
|
+
}
|
|
518
|
+
if (!Array.isArray(events) || !events.length) return [];
|
|
519
|
+
|
|
520
|
+
const identityMap = profiles ? buildIdentityMap(profiles) : undefined;
|
|
521
|
+
let minMs = Infinity, maxMs = -Infinity;
|
|
522
|
+
const matches = [];
|
|
523
|
+
for (const e of events) {
|
|
524
|
+
if (!e) continue;
|
|
525
|
+
const ms = toMs(e.time);
|
|
526
|
+
if (!Number.isFinite(ms)) continue;
|
|
527
|
+
if (ms < minMs) minMs = ms;
|
|
528
|
+
if (ms > maxMs) maxMs = ms;
|
|
529
|
+
if (e.event !== event) continue;
|
|
530
|
+
const uid = resolveUserId(e, identityMap);
|
|
531
|
+
if (!uid) continue;
|
|
532
|
+
matches.push({ uid, ms });
|
|
533
|
+
}
|
|
534
|
+
if (!Number.isFinite(minMs)) return [];
|
|
535
|
+
|
|
536
|
+
const intervalMs = intervalDays * 86400_000;
|
|
537
|
+
const unitMs = unitSeconds * 1000;
|
|
538
|
+
// UTC midnight of the first event's day — deterministic from_date stand-in.
|
|
539
|
+
const fromMs = Math.floor(minMs / 86400_000) * 86400_000;
|
|
540
|
+
const numIntervals = Math.floor((maxMs - fromMs) / intervalMs) + 1;
|
|
541
|
+
// ceil — addiction_units_per_interval (unit.c:108-113)
|
|
542
|
+
const histLen = Math.ceil((intervalDays * 86400) / unitSeconds);
|
|
543
|
+
|
|
544
|
+
// Per-(user, interval) rolling histories (addiction_query.cpp:363-374).
|
|
545
|
+
// One Map per interval mirrors ARB's calloc'd per-interval history slabs.
|
|
546
|
+
/** @type {Array<Map<string, { count: number, lastCountedMs: number }>>} */
|
|
547
|
+
const perInterval = Array.from({ length: numIntervals }, () => new Map());
|
|
548
|
+
matches.sort((a, b) => a.ms - b.ms);
|
|
549
|
+
for (const { uid, ms } of matches) {
|
|
550
|
+
// forward tiling — uniform_intervals_for (uniform_intervals.h:48-58)
|
|
551
|
+
const idx = Math.floor((ms - fromMs) / intervalMs);
|
|
552
|
+
const histories = perInterval[idx];
|
|
553
|
+
const h = histories.get(uid);
|
|
554
|
+
if (!h) {
|
|
555
|
+
// calloc-zeroed history → first qualifying event always counts
|
|
556
|
+
histories.set(uid, { count: 1, lastCountedMs: ms });
|
|
557
|
+
} else if (ms >= h.lastCountedMs + unitMs) {
|
|
558
|
+
h.count++;
|
|
559
|
+
h.lastCountedMs = ms;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
const out = [];
|
|
564
|
+
for (let i = 0; i < numIntervals; i++) {
|
|
565
|
+
const histogram = new Array(histLen).fill(0);
|
|
566
|
+
for (const { count } of perInterval[i].values()) {
|
|
567
|
+
// count > 0 by construction — zero-count users never enter the
|
|
568
|
+
// map, matching ARB's no-zero-bucket omission (:546-573).
|
|
569
|
+
histogram[count - 1] += 1;
|
|
570
|
+
}
|
|
571
|
+
out.push({ interval: formatBucket(fromMs + i * intervalMs, 'day'), histogram });
|
|
572
|
+
}
|
|
573
|
+
return out;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Count events, optionally filtered by event name and a `where` object —
|
|
578
|
+
* Mixpanel's plain ACTION_TYPE_COUNT (action.h:14; each qualifying event
|
|
579
|
+
* increments the interval count by one — normal_query.cpp
|
|
580
|
+
* `result->count += q->upsampling_factor`). `where` string comparison is
|
|
581
|
+
* case-INSENSITIVE per the WHERE-filter rulebook (see coerce.js).
|
|
582
|
+
*
|
|
583
|
+
* @param {Object[]} events
|
|
584
|
+
* @param {Object} [options]
|
|
585
|
+
* @param {string} [options.event] restrict to this event name
|
|
586
|
+
* @param {Object<string, *>} [options.where] `{ prop: value | { op, value } }`
|
|
587
|
+
* @returns {number}
|
|
588
|
+
*/
|
|
589
|
+
export function countEvents(events, options = {}) {
|
|
590
|
+
const { event, where } = options;
|
|
591
|
+
let count = 0;
|
|
592
|
+
for (const e of events) {
|
|
593
|
+
if (!e || (event && e.event !== event)) continue;
|
|
594
|
+
if (!matchesWhere(e, where)) continue;
|
|
595
|
+
count++;
|
|
596
|
+
}
|
|
597
|
+
return count;
|
|
598
|
+
}
|