@ak--47/dungeon-master 1.5.4 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +1 -1
- package/.claude/skills/create-project/provision.mjs +57 -10
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +183 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +2 -1
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: analyze-soup
|
|
3
3
|
description: Use when investigating TimeSoup parameters, diagnosing event-distribution shape, or comparing soup configs — runs a dungeon locally and analyzes time distribution at week/day/hour/minute granularities, producing a soup-analysis.md diagnostic report.
|
|
4
|
-
argument-hint: [dungeon path, e.g. dungeons/
|
|
4
|
+
argument-hint: [dungeon path, e.g. dungeons/technical/simplest.js]
|
|
5
5
|
model: claude-opus-4-6
|
|
6
6
|
effort: max
|
|
7
7
|
---
|
|
@@ -10,7 +10,7 @@ effort: max
|
|
|
10
10
|
|
|
11
11
|
Run a dungeon and analyze the time distribution of generated events to evaluate TimeSoup parameters.
|
|
12
12
|
|
|
13
|
-
**Dungeon file:** `$ARGUMENTS` (default: `dungeons/
|
|
13
|
+
**Dungeon file:** `$ARGUMENTS` (default: `dungeons/technical/simplest.js`)
|
|
14
14
|
|
|
15
15
|
## Step 1: Run the Dungeon
|
|
16
16
|
|
|
@@ -32,10 +32,19 @@ Wait for generation to complete. Note the event count and EPS.
|
|
|
32
32
|
|
|
33
33
|
Run these DuckDB queries against the generated JSONL file. Use `duckdb` CLI.
|
|
34
34
|
|
|
35
|
+
**All bucketing is UTC — do not convert timezones.** TimeSoup applies its
|
|
36
|
+
day-of-week / hour-of-day weights in UTC, the engine's calendar-day logic
|
|
37
|
+
(`avgActiveDaysPerUser`, lifecycle periods, session midnight splits) is UTC,
|
|
38
|
+
and `/create-project` pins Mixpanel projects to UTC. Casting `time::timestamp`
|
|
39
|
+
on the ISO strings keeps the UTC wall time. Converting to a local zone (e.g.
|
|
40
|
+
`America/Los_Angeles`, −8h) shifts late-evening UTC events onto the previous
|
|
41
|
+
calendar day — the DOW histogram you'd analyze would not be the one the soup
|
|
42
|
+
generated or the one Mixpanel reports.
|
|
43
|
+
|
|
35
44
|
### 2a. Week over Week
|
|
36
45
|
```bash
|
|
37
46
|
duckdb -c "
|
|
38
|
-
SELECT date_trunc('week',
|
|
47
|
+
SELECT date_trunc('week', time::timestamp) as week,
|
|
39
48
|
count(*) as events
|
|
40
49
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
41
50
|
GROUP BY 1 ORDER BY 1;
|
|
@@ -45,7 +54,7 @@ GROUP BY 1 ORDER BY 1;
|
|
|
45
54
|
### 2b. Day over Day
|
|
46
55
|
```bash
|
|
47
56
|
duckdb -c "
|
|
48
|
-
SELECT date_trunc('day',
|
|
57
|
+
SELECT date_trunc('day', time::timestamp) as day,
|
|
49
58
|
count(*) as events
|
|
50
59
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
51
60
|
GROUP BY 1 ORDER BY 1;
|
|
@@ -55,10 +64,10 @@ GROUP BY 1 ORDER BY 1;
|
|
|
55
64
|
### 2c. Hour over Hour (last 7 days only)
|
|
56
65
|
```bash
|
|
57
66
|
duckdb -c "
|
|
58
|
-
SELECT date_trunc('hour',
|
|
67
|
+
SELECT date_trunc('hour', time::timestamp) as hour,
|
|
59
68
|
count(*) as events
|
|
60
69
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
61
|
-
WHERE
|
|
70
|
+
WHERE time::timestamp > (SELECT max(time::timestamp) - interval '7 days' FROM read_json_auto('./data/soup-analysis-EVENTS.json'))
|
|
62
71
|
GROUP BY 1 ORDER BY 1;
|
|
63
72
|
"
|
|
64
73
|
```
|
|
@@ -66,10 +75,10 @@ GROUP BY 1 ORDER BY 1;
|
|
|
66
75
|
### 2d. Minute over Minute (last 24 hours only)
|
|
67
76
|
```bash
|
|
68
77
|
duckdb -c "
|
|
69
|
-
SELECT date_trunc('minute',
|
|
78
|
+
SELECT date_trunc('minute', time::timestamp) as minute,
|
|
70
79
|
count(*) as events
|
|
71
80
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
72
|
-
WHERE
|
|
81
|
+
WHERE time::timestamp > (SELECT max(time::timestamp) - interval '1 day' FROM read_json_auto('./data/soup-analysis-EVENTS.json'))
|
|
73
82
|
GROUP BY 1 ORDER BY 1;
|
|
74
83
|
"
|
|
75
84
|
```
|
|
@@ -78,14 +87,14 @@ GROUP BY 1 ORDER BY 1;
|
|
|
78
87
|
```bash
|
|
79
88
|
duckdb -c "
|
|
80
89
|
WITH daily AS (
|
|
81
|
-
SELECT date_trunc('day',
|
|
90
|
+
SELECT date_trunc('day', time::timestamp) as day, count(*) as events
|
|
82
91
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
83
92
|
GROUP BY 1
|
|
84
93
|
),
|
|
85
94
|
hourly AS (
|
|
86
|
-
SELECT date_trunc('hour',
|
|
95
|
+
SELECT date_trunc('hour', time::timestamp) as hour, count(*) as events
|
|
87
96
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
88
|
-
WHERE
|
|
97
|
+
WHERE time::timestamp > (SELECT max(time::timestamp) - interval '7 days' FROM read_json_auto('./data/soup-analysis-EVENTS.json'))
|
|
89
98
|
GROUP BY 1
|
|
90
99
|
)
|
|
91
100
|
SELECT 'daily' as granularity,
|
|
@@ -112,7 +121,7 @@ FROM hourly;
|
|
|
112
121
|
```bash
|
|
113
122
|
duckdb -c "
|
|
114
123
|
WITH daily AS (
|
|
115
|
-
SELECT date_trunc('day',
|
|
124
|
+
SELECT date_trunc('day', time::timestamp) as day, count(*) as events
|
|
116
125
|
FROM read_json_auto('./data/soup-analysis-EVENTS.json')
|
|
117
126
|
GROUP BY 1
|
|
118
127
|
)
|
|
@@ -51,7 +51,14 @@ Out of scope (hand off to `write-hooks`):
|
|
|
51
51
|
For an encyclopedia of hook patterns, recipes, and real-world examples:
|
|
52
52
|
see `HOOKS.md` at the project root.
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
> ⚠️ **Silently-ignored config keys — do not use.** The validator STRIPS these
|
|
55
|
+
> five keys before generation: `subscription`, `attribution`, `geo`,
|
|
56
|
+
> `features`, `anomalies`. They produce **zero data** — only a verbose-mode
|
|
57
|
+
> warning. A dungeon that "configures" churn via `subscription:` or campaign
|
|
58
|
+
> lift via `attribution:` is configuring nothing, and the omission is
|
|
59
|
+
> invisible at smoke-test scale. Recreate those behaviors with hooks via
|
|
60
|
+
> `/write-hooks` (churn cohorts, UTM biasing, geo weighting, feature-flag
|
|
61
|
+
> cohorts, anomaly windows all have recipes in `HOOKS.md`).
|
|
55
62
|
|
|
56
63
|
## Reference reading
|
|
57
64
|
|
|
@@ -62,8 +69,10 @@ Before writing any code, scan:
|
|
|
62
69
|
with full JSDoc. **Treat this as the source of truth.**
|
|
63
70
|
- `lib/utils/utils.js` — `pickAWinner`, `weighNumRange`, `initChance`, `exhaust`,
|
|
64
71
|
`takeSome` for property value distributions
|
|
65
|
-
- `dungeons/vertical/sass.js` — B2B reference dungeon with full identity model
|
|
66
|
-
- `dungeons/
|
|
72
|
+
- `dungeons/vertical/sass/sass.js` — B2B reference dungeon with full identity model
|
|
73
|
+
- `dungeons/vertical/ecommerce/ecommerce.js` — canonical consumer exemplar:
|
|
74
|
+
schema, funnels, stories export, and full hook layout (what your schema
|
|
75
|
+
becomes after `/write-hooks`)
|
|
67
76
|
- `dungeons/technical/identity-model-verify.js` — minimal identity-model fixture
|
|
68
77
|
|
|
69
78
|
## File structure
|
|
@@ -88,6 +97,7 @@ import * as v from "ak-tools";
|
|
|
88
97
|
* APP: <2-4 line description: what users do, core flow, monetization>
|
|
89
98
|
* SCALE: <numUsers> users, ~<numEvents> events, <numDays> days (<start> → <end>)
|
|
90
99
|
* CORE LOOP: <event1> → <event2> → <event3> → ...
|
|
100
|
+
* VALUE MOMENT: <the one event representing realized value — lifecycle hooks anchor here>
|
|
91
101
|
*
|
|
92
102
|
* EVENTS (N):
|
|
93
103
|
* <event name (weight)> > ... (sorted by weight desc)
|
|
@@ -160,7 +170,7 @@ KNOBS (extracted tunable constants — timing, thresholds, multipliers),
|
|
|
160
170
|
HOOK STATE (module-level Maps/Sets used across users), and HELPER FUNCTIONS
|
|
161
171
|
(per-type handlers like `handleEventHooks`, `handleEverythingHooks`).
|
|
162
172
|
`config.hook` becomes a thin dispatcher delegating to the helpers. See
|
|
163
|
-
`dungeons/vertical/ecommerce.js` as the canonical exemplar.
|
|
173
|
+
`dungeons/vertical/ecommerce/ecommerce.js` as the canonical exemplar.
|
|
164
174
|
|
|
165
175
|
## Required components
|
|
166
176
|
|
|
@@ -196,6 +206,30 @@ When using experiments, include `$experiment_started` in the events array with
|
|
|
196
206
|
}}
|
|
197
207
|
```
|
|
198
208
|
|
|
209
|
+
**Design the vocabulary for the analyses hooks will target (v1.6):**
|
|
210
|
+
|
|
211
|
+
- **Session-friendly next-events.** Flows (Sankey) reads collapse when an
|
|
212
|
+
anchor event can plausibly be followed by 15 different event types — every
|
|
213
|
+
branch gets a sliver and no engineered path clears the top level. For each
|
|
214
|
+
high-traffic anchor (post-login, post-search, post-add-to-cart), keep the
|
|
215
|
+
realistic next-event vocabulary to **≤5–6 event types**; if the app
|
|
216
|
+
genuinely has more surfaces, give the rare ones weight 1 so the top paths
|
|
217
|
+
stay legible. A path-share hook needs ~20–25% share to survive a Sankey's
|
|
218
|
+
top-3 — impossible against a 15-way fan-out.
|
|
219
|
+
- **Designate a value moment.** Pick ONE event that represents realized value
|
|
220
|
+
(first workout logged, order delivered, report generated) and record it in
|
|
221
|
+
the OVERVIEW block as `VALUE MOMENT: <event>`. Lifecycle stories
|
|
222
|
+
(dormancy → resurrection waves) anchor on the value moment — hooks gate or
|
|
223
|
+
inject THAT event, not generic page views. A schema with no obvious value
|
|
224
|
+
moment forces `write-hooks` to invent one badly.
|
|
225
|
+
- **Hidden-event hygiene.** Heartbeat/telemetry events (`ping`,
|
|
226
|
+
`app foregrounded`, `position updated`) at weight ≥5 drown every Sankey
|
|
227
|
+
level — every top path becomes anchor → heartbeat → heartbeat. Mixpanel
|
|
228
|
+
Flows (and the emulator's `topPaths` via `hiddenEvents`) can hide them at
|
|
229
|
+
query time, but the default view is what demos show. If realism demands
|
|
230
|
+
heartbeats, keep them at weight 1–2 and list them in the OVERVIEW so
|
|
231
|
+
`write-hooks` knows to pass `hiddenEvents` in flows stories.
|
|
232
|
+
|
|
199
233
|
### 2. Funnels (3–6)
|
|
200
234
|
|
|
201
235
|
- First funnel: includes `isFirstEvent` AND has `isAuthEvent: true` on the
|
|
@@ -237,6 +271,32 @@ event.
|
|
|
237
271
|
|
|
238
272
|
**Funnels representing loops need `reentry: true`.** Any funnel named "X loop" / "X cycle" / "session" / per-instance recurring behavior must declare `reentry: true`. Without it, the engine emits one sequence per user and downstream "power user" / "daily active" cohorts have no behavioral signal to bin on.
|
|
239
273
|
|
|
274
|
+
**Structural trend engineering — duplicate funnels instead of reaching for
|
|
275
|
+
hooks.** Not every trend needs a hook: initial conditions can raise or lower
|
|
276
|
+
a metric by themselves. The funnel array is a set of knobs — duplicate a
|
|
277
|
+
funnel and swap steps, props, `conversionRate`, `timeToConvert`, or `weight`
|
|
278
|
+
to architect a comparison directly into the schema:
|
|
279
|
+
|
|
280
|
+
- **Exclusion-step story**: Mixpanel funnels can exclude users who did
|
|
281
|
+
`foo` between A and B. There's no `excludeSteps` config — model it with
|
|
282
|
+
two funnels, `[A, B, C]` and `[A, B, foo, C]`, giving the second a lower
|
|
283
|
+
`conversionRate`. The report shows the conversion dip for the
|
|
284
|
+
detour-takers with zero hook code.
|
|
285
|
+
- **Segment-split story**: two copies of the same sequence with different
|
|
286
|
+
`props` (`{ checkout_version: "v1" }` vs `"v2"`) and different
|
|
287
|
+
`conversionRate` — an A/B story without the experiment machinery (use
|
|
288
|
+
`experiment` when you want `$experiment_started` + variant reports).
|
|
289
|
+
- **Speed story**: same sequence, different `timeToConvert`, split by a
|
|
290
|
+
funnel-level prop — time-to-convert deltas appear structurally.
|
|
291
|
+
- **Mix-shift story**: `weight` controls how often each variant funnel is
|
|
292
|
+
chosen; a heavy-weighted low-converting path drags the blended rate.
|
|
293
|
+
|
|
294
|
+
Prefer structure when the story is a *between-path comparison* (this path
|
|
295
|
+
converts worse / slower than that one). Reach for hooks when the story is a
|
|
296
|
+
*within-cohort behavior* (these users do more of X over time, this segment's
|
|
297
|
+
values differ). Structural trends are cheaper to verify — the knob IS the
|
|
298
|
+
expected value.
|
|
299
|
+
|
|
240
300
|
### 3. SuperProps (2–3)
|
|
241
301
|
|
|
242
302
|
Properties present on EVERY event. Common picks: `Plan`, `Region`, `Platform`,
|
|
@@ -554,7 +614,7 @@ When designing event properties, always consider which Mixpanel type best repres
|
|
|
554
614
|
app/customer, then **create `dungeons/user/<name>/` if it doesn't already exist**
|
|
555
615
|
(`mkdir -p dungeons/user/<name>`) and write the dungeon to
|
|
556
616
|
`dungeons/user/<name>/<name>.js` (e.g. `dungeons/user/acme/acme.js`). Folder and
|
|
557
|
-
file share the name, matching `
|
|
617
|
+
file share the name, matching the vertical layout (`ecommerce/ecommerce.js`).
|
|
558
618
|
|
|
559
619
|
This keeps `dungeons/user/` organized — EVERYTHING about this dungeon lives in
|
|
560
620
|
the same folder: `hook-results.md` + `hook-query-log.txt` +
|
|
@@ -25,7 +25,7 @@ in order:
|
|
|
25
25
|
1. **createProject** — name derived from the dungeon's `OVERVIEW` (`NAME:` line), region `US`, timezone `UTC` (set as a follow-up by the endpoint).
|
|
26
26
|
2. **mintServiceAccount** — `admin`, expires `+30 days`, scoped to the new project. This is what the dungeon uses to **send** data.
|
|
27
27
|
3. **addGroupKey** — one per `groupKeys` entry in the dungeon (`property_name` + a titleized `display_name`). Skipped if the dungeon has no group keys.
|
|
28
|
-
4. **setBusinessContext** — markdown built from the dungeon's `OVERVIEW` + `HOOK STORIES` comment
|
|
28
|
+
4. **setBusinessContext** — markdown built from the dungeon's `OVERVIEW` comment block plus the `stories` named export (per story: `narrative` + `mixpanelReport` + intentional deviations); dungeons without stories fall back to the `HOOK STORIES` comment scrape (via the package's `extractComments`). Plus an events/funnels/props/group-keys summary, capped at 50k chars. The dry-run plan prints which source was used.
|
|
29
29
|
5. **write-back** — replaces the dungeon's `credentials: { … }` block with `{ token, projectId, serviceAccount, serviceSecret, region }`.
|
|
30
30
|
|
|
31
31
|
**Always creates a fresh project.** Re-running mints a new project and overwrites
|
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
* 1. createProject (sets timezone UTC as a follow-up)
|
|
11
11
|
* 2. mintServiceAccount (admin, +30d) — scoped to the new project
|
|
12
12
|
* 3. addGroupKey (only if the dungeon declares groupKeys)
|
|
13
|
-
* 4. setBusinessContext (OVERVIEW +
|
|
13
|
+
* 4. setBusinessContext (OVERVIEW + the dungeon's `stories` export
|
|
14
|
+
* (narrative + mixpanelReport per story) — falls back to the HOOK
|
|
15
|
+
* STORIES comment scrape when the dungeon has no stories — + schema summary)
|
|
14
16
|
* 5. write `credentials: { token, projectId, serviceAccount, serviceSecret, region }`
|
|
15
17
|
* back into the dungeon (gitignored user dungeons — plaintext is fine)
|
|
16
18
|
*
|
|
@@ -27,7 +29,7 @@
|
|
|
27
29
|
|
|
28
30
|
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
|
29
31
|
import path, { dirname, resolve } from 'path';
|
|
30
|
-
import { fileURLToPath } from 'url';
|
|
32
|
+
import { fileURLToPath, pathToFileURL } from 'url';
|
|
31
33
|
import dotenv from 'dotenv';
|
|
32
34
|
import { loadFromFile, extractComments } from '../../../index.js';
|
|
33
35
|
|
|
@@ -57,12 +59,26 @@ if (!existsSync(dungeonPath)) fail(`dungeon file not found: ${dungeonPath}`);
|
|
|
57
59
|
const config = await loadFromFile(dungeonPath);
|
|
58
60
|
const comments = extractComments(dungeonPath);
|
|
59
61
|
|
|
62
|
+
// stories is a NAMED export of JS dungeons (v1.6 story-spec) — the verified
|
|
63
|
+
// behavior contract. Preferred over the comment scrape for business context.
|
|
64
|
+
let stories = null;
|
|
65
|
+
if (/\.(js|mjs)$/i.test(dungeonPath)) {
|
|
66
|
+
try {
|
|
67
|
+
const mod = await import(pathToFileURL(dungeonPath).href);
|
|
68
|
+
if (Array.isArray(mod.stories) && mod.stories.length) stories = mod.stories;
|
|
69
|
+
} catch {
|
|
70
|
+
// loadFromFile above already imported the module; an error here would
|
|
71
|
+
// have surfaced there. Fall back to the comment scrape regardless.
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
60
75
|
const name = deriveName(comments, dungeonPath);
|
|
61
76
|
const groupKeys = Array.isArray(config.groupKeys)
|
|
62
77
|
? config.groupKeys.filter(Boolean).map(([prop]) => ({ property_name: prop, display_name: titleize(prop) }))
|
|
63
78
|
: [];
|
|
64
79
|
const saName = `${slug(name)}-dungeon-sa`.slice(0, 64);
|
|
65
|
-
const
|
|
80
|
+
const ctxSource = stories ? `stories export (${stories.length} stories)` : 'comment scrape';
|
|
81
|
+
const content = buildContext(name, config, comments, groupKeys, stories);
|
|
66
82
|
|
|
67
83
|
if (dryRun) {
|
|
68
84
|
printPlan();
|
|
@@ -182,13 +198,41 @@ function slug(s) {
|
|
|
182
198
|
|
|
183
199
|
function isoInDays(days) {
|
|
184
200
|
const ms = Date.now() + days * 24 * 60 * 60 * 1000;
|
|
185
|
-
|
|
201
|
+
// Mixpanel's createServiceAccount requires %Y-%m-%dT%H:%M:%SZ — seconds
|
|
202
|
+
// precision, NO milliseconds. toISOString() emits ".sssZ", which the API
|
|
203
|
+
// rejects ("Expiration date … does not match %Y-%m-%dT%H:%M:%SZ").
|
|
204
|
+
return new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
|
186
205
|
}
|
|
187
206
|
|
|
188
|
-
function buildContext(name, config, comments, groupKeys) {
|
|
207
|
+
function buildContext(name, config, comments, groupKeys, stories) {
|
|
189
208
|
const parts = [`# ${name}`, ''];
|
|
190
209
|
if (comments.overview) parts.push(comments.overview, '');
|
|
191
|
-
|
|
210
|
+
|
|
211
|
+
// Preferred source: the stories export — each story's narrative is the
|
|
212
|
+
// human-readable behavior and mixpanelReport (free-form object) points at
|
|
213
|
+
// the report where it shows. Comment scrape is the fallback for dungeons
|
|
214
|
+
// without stories (schema-only or pre-1.6).
|
|
215
|
+
if (stories?.length) {
|
|
216
|
+
parts.push('## Engineered Behaviors', '');
|
|
217
|
+
parts.push(`${stories.length} machine-verified story patterns (from the dungeon's \`stories\` export):`, '');
|
|
218
|
+
for (const s of stories) {
|
|
219
|
+
const arch = s.archetype ? ` — ${s.archetype}` : '';
|
|
220
|
+
parts.push(`### ${s.id}${arch}`, '');
|
|
221
|
+
if (s.narrative) parts.push(String(s.narrative).trim(), '');
|
|
222
|
+
if (s.mixpanelReport && typeof s.mixpanelReport === 'object') {
|
|
223
|
+
parts.push('How to see it in Mixpanel:');
|
|
224
|
+
for (const [k, v] of Object.entries(s.mixpanelReport)) {
|
|
225
|
+
parts.push(`- **${k}**: ${typeof v === 'string' ? v : JSON.stringify(v)}`);
|
|
226
|
+
}
|
|
227
|
+
parts.push('');
|
|
228
|
+
}
|
|
229
|
+
if (Array.isArray(s.intentionalDeviations) && s.intentionalDeviations.length) {
|
|
230
|
+
parts.push('Notes:', ...s.intentionalDeviations.map((d) => `- ${d}`), '');
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
} else if (comments.hookStories) {
|
|
234
|
+
parts.push('## Engineered Behaviors', '', comments.hookStories, '');
|
|
235
|
+
}
|
|
192
236
|
|
|
193
237
|
parts.push('## Schema', '');
|
|
194
238
|
const events = config.events || [];
|
|
@@ -252,7 +296,7 @@ function printPlan() {
|
|
|
252
296
|
console.log(`region: ${REGION} timezone: UTC`);
|
|
253
297
|
console.log(`service acct: ${saName} (role admin, expires +${SA_TTL_DAYS}d)`);
|
|
254
298
|
console.log(`group keys: ${groupKeys.length ? groupKeys.map((g) => `${g.property_name} → "${g.display_name}"`).join(', ') : '(none)'}`);
|
|
255
|
-
console.log(`business ctx: ${content.length} chars`);
|
|
299
|
+
console.log(`business ctx: ${content.length} chars (source: ${ctxSource})`);
|
|
256
300
|
console.log('');
|
|
257
301
|
console.log('would POST: createProject → mintServiceAccount' + (groupKeys.length ? ' → addGroupKey' : '') + ' → setBusinessContext');
|
|
258
302
|
console.log('then write credentials back into the dungeon.');
|
|
@@ -268,11 +312,14 @@ function printSummary() {
|
|
|
268
312
|
console.log(`region: ${REGION} timezone: UTC`);
|
|
269
313
|
console.log(`service acct: ${sa ? `${sa.username} (role ${sa.role}, expires ${sa.expires})` : '(FAILED — see warnings)'}`);
|
|
270
314
|
if (groupKeys.length) {
|
|
271
|
-
|
|
272
|
-
|
|
315
|
+
// Defensive: the API's `added`/`skipped` entries may be objects, strings, or
|
|
316
|
+
// null. Never let summary formatting throw — it would swallow real warnings.
|
|
317
|
+
const fmt = (g) => (g == null ? null : typeof g === 'string' ? g : (g.property_name || g.name || null));
|
|
318
|
+
const added = (groupKeyResult?.added || []).map(fmt).filter(Boolean).join(', ') || '(none)';
|
|
319
|
+
const skipped = (groupKeyResult?.skipped || []).map(fmt).filter(Boolean).join(', ') || '(none)';
|
|
273
320
|
console.log(`group keys: added [${added}] skipped [${skipped}]`);
|
|
274
321
|
}
|
|
275
|
-
console.log(`business ctx: ${content.length} chars uploaded`);
|
|
322
|
+
console.log(`business ctx: ${content.length} chars uploaded (source: ${ctxSource})`);
|
|
276
323
|
console.log(`credentials: ${wroteBack ? 'written back into dungeon ✓' : 'NOT written (see warnings)'}`);
|
|
277
324
|
if (warnings.length) {
|
|
278
325
|
console.log('');
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: verify-dungeon
|
|
3
|
-
description: Use when a dungeon's hooks need verification before pushing data to Mixpanel — runs the dungeon at full scale, validates schema integrity (catches flag-stamping),
|
|
4
|
-
argument-hint: [dungeon path(s), e.g. dungeons/gaming.js or dungeons/fintech.js]
|
|
3
|
+
description: Use when a dungeon's hooks need verification before pushing data to Mixpanel — runs the dungeon at full scale, evaluates the dungeon's `stories` export with the mechanical story runner (verify-stories.mjs), validates schema integrity (catches flag-stamping), and reports per-hook NAILED/STRONG/WEAK/NONE/INVERSE verdicts. LLM investigation is reserved for failures and duckdb-type assertions.
|
|
4
|
+
argument-hint: [dungeon path(s), e.g. dungeons/vertical/gaming/gaming.js or dungeons/vertical/fintech/fintech.js]
|
|
5
5
|
model: claude-opus-4-6
|
|
6
6
|
effort: max
|
|
7
7
|
---
|
|
8
8
|
|
|
9
9
|
# Verify Dungeon
|
|
10
10
|
|
|
11
|
-
Verify a dungeon at full scale: validate schema integrity,
|
|
11
|
+
Verify a dungeon at full scale: run the story runner (`scripts/verify-stories.mjs`) as the primary mechanical check, validate schema integrity, investigate only what the runner can't settle (failures + `duckdb`-type assertions), write a single consolidated `hook-results.md`.
|
|
12
12
|
|
|
13
|
-
**Dungeon file(s):** `$ARGUMENTS` — single path, multiple space-separated paths, or glob pattern. In batch mode, process each dungeon sequentially through Steps 1–
|
|
13
|
+
**Dungeon file(s):** `$ARGUMENTS` — single path, multiple space-separated paths, or glob pattern. In batch mode, process each dungeon sequentially through Steps 1–5, then write one consolidated report in Step 7. Use a unique `name` prefix per dungeon (e.g., `verify-fintech`, `verify-gaming`) so output files don't collide.
|
|
14
14
|
|
|
15
15
|
## Reference files
|
|
16
16
|
|
|
@@ -28,9 +28,10 @@ Also: `HOOKS.md` (recipe encyclopedia) and `types.d.ts` (source of truth for hoo
|
|
|
28
28
|
|
|
29
29
|
Read the dungeon at `$ARGUMENTS`. If it's a bare filename (no `/`), check `dungeons/`. Find:
|
|
30
30
|
|
|
31
|
-
1. **The `
|
|
32
|
-
2. **The
|
|
33
|
-
3. **
|
|
31
|
+
1. **The `stories` named export** — the machine-checkable contract (see `lib/templates/story-spec.schema.json` and the `DungeonStory` typedef in `types.d.ts`). If present, the story runner in Step 3 does the heavy lifting. If absent (legacy dungeon), the full per-hook flow in Step 5 applies to every hook.
|
|
32
|
+
2. **The `hook:` function** — read the full body
|
|
33
|
+
3. **The documentation comment block** describing engineered patterns
|
|
34
|
+
4. **Module-level closure state** (Map / Set / tracking variables defined outside the hook function but used inside)
|
|
34
35
|
|
|
35
36
|
For each hook/pattern, catalog:
|
|
36
37
|
- Hook number and name (e.g., "Hook #1: Compass users have 3x quest completion")
|
|
@@ -52,7 +53,7 @@ node scripts/verify-runner.mjs <dungeon-path> <run-name>
|
|
|
52
53
|
|
|
53
54
|
Example:
|
|
54
55
|
```bash
|
|
55
|
-
node scripts/verify-runner.mjs dungeons/vertical/gaming.js verify-gaming
|
|
56
|
+
node scripts/verify-runner.mjs dungeons/vertical/gaming/gaming.js verify-gaming
|
|
56
57
|
```
|
|
57
58
|
|
|
58
59
|
Full-fidelity runs can take minutes (50K+ user dungeons). Plan accordingly — kick off the run, do other reading, return when the file lands. If a run takes longer than your budget, report that as a finding ("dungeon too large to verify in current session") rather than falling back to `--small`.
|
|
@@ -65,7 +66,27 @@ Full-fidelity runs can take minutes (50K+ user dungeons). Plan accordingly — k
|
|
|
65
66
|
|
|
66
67
|
For batched output (>2M events), the runner writes `<run-name>-EVENTS-part-*.json`. See [sql-recipes.md "Multi-part EVENTS files"](references/sql-recipes.md#multi-part-events-files-batch-mode).
|
|
67
68
|
|
|
68
|
-
### Step 3:
|
|
69
|
+
### Step 3: Run the story runner (primary mechanical check)
|
|
70
|
+
|
|
71
|
+
For any dungeon with a `stories` export, the runner — not the LLM — computes the verdicts:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
node scripts/verify-stories.mjs <dungeon-path> --data-prefix <run-name>
|
|
75
|
+
node scripts/verify-stories.mjs <dungeon-path> --data-prefix <run-name> --json # machine-readable, for hook-results.md
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The runner streams the shards from Step 2, evaluates every assertion against its declared `target` / `floor` / `minCohort`, substitutes `{{PREFIX}}` into `duckdb`-type assertions and shells them out, enforces hook coverage (every numbered hook in the HOOK STORIES comment block must be targeted by at least one story), and prints a five-tier verdict table. Exit code is non-zero when any story lands WEAK / NONE / INVERSE or coverage is incomplete.
|
|
79
|
+
|
|
80
|
+
**Verdicts are computed, not judged.** They include the population floor: a cohort smaller than the assertion's `minCohort` caps at WEAK — a 12-user cohort can no longer score NAILED regardless of how clean its ratio looks. See [report-format.md "Verdict criteria"](references/report-format.md#verdict-criteria-5-tier) for the mechanical definitions.
|
|
81
|
+
|
|
82
|
+
**What the LLM investigates after this step — and nothing else:**
|
|
83
|
+
1. **Stories below STRONG** (WEAK / NONE / INVERSE) — root-cause via Step 5's decision table. A miss means fixing the hook or the assertion's derivation, never relaxing the number to match output.
|
|
84
|
+
2. **`duckdb`-type assertions** — the runner executes them but can't interpret bespoke shapes; sanity-check their output against the story narrative.
|
|
85
|
+
3. **Dungeons without a `stories` export** — legacy fallback: full per-hook flow (Step 5) for every documented hook.
|
|
86
|
+
|
|
87
|
+
Do NOT re-derive verdicts the runner already computed as passing. `hook-results.md` (Step 7) renders the runner's JSON.
|
|
88
|
+
|
|
89
|
+
### Step 4: Validate schema (BEFORE per-hook checks)
|
|
69
90
|
|
|
70
91
|
Catches hooks that introduce undeclared columns (flag stamping). For each unique event type, compare actual columns against config-declared properties. See [sql-recipes.md "Schema validation queries"](references/sql-recipes.md#schema-validation-queries) for the SQL and the expected-schema source table.
|
|
71
92
|
|
|
@@ -75,7 +96,9 @@ Catches hooks that introduce undeclared columns (flag stamping). For each unique
|
|
|
75
96
|
|
|
76
97
|
If any event type has SCHEMA-FAIL, flag it prominently in the report header with specific remediation: which hook line adds the property and how to remove it while preserving the intended pattern.
|
|
77
98
|
|
|
78
|
-
### Step
|
|
99
|
+
### Step 5: Investigate failures (and legacy no-stories dungeons)
|
|
100
|
+
|
|
101
|
+
Applies only to the investigation targets from Step 3 — failing stories, `duckdb`-type assertions, and dungeons with no `stories` export.
|
|
79
102
|
|
|
80
103
|
**Decision: emulator vs DuckDB**
|
|
81
104
|
|
|
@@ -84,14 +107,22 @@ If any event type has SCHEMA-FAIL, flag it prominently in the report header with
|
|
|
84
107
|
| Funnel completion / step conversion | `emulateBreakdown({type: 'funnelFrequency'})` |
|
|
85
108
|
| Insights frequency-distribution | `emulateBreakdown({type: 'frequencyByFrequency'})` |
|
|
86
109
|
| Avg(prop) by per-user count(B) | `emulateBreakdown({type: 'aggregatePerUser'})` |
|
|
87
|
-
| Funnel TTC by segment | `emulateBreakdown({type: 'timeToConvert'})` |
|
|
110
|
+
| Funnel TTC by segment (steps-based or funnel-based) | `emulateBreakdown({type: 'timeToConvert'})` |
|
|
88
111
|
| First/last touch attribution | `emulateBreakdown({type: 'attributedBy'})` |
|
|
89
|
-
| Retention curves | `emulateBreakdown({type: 'retention'})` |
|
|
112
|
+
| Retention curves — birth or compounded | `emulateBreakdown({type: 'retention'})` (`compounded: true` for "DAU coming back") |
|
|
90
113
|
| Per-session metrics | `emulateBreakdown({type: 'sessionMetrics'})` |
|
|
114
|
+
| Lifecycle (new / retained / resurrected / dormant) | `emulateBreakdown({type: 'lifecycle'})` |
|
|
115
|
+
| Flows / top paths (Sankey) | `emulateBreakdown({type: 'topPaths'})` |
|
|
116
|
+
| Event totals segmented by property | `emulateBreakdown({type: 'eventBreakdown'})` (`countType: 'general' \| 'sessions'`) |
|
|
117
|
+
| Uniques per segment | `emulateBreakdown({type: 'uniques'})` |
|
|
118
|
+
| COUNT DISTINCT of a property + top values | `emulateBreakdown({type: 'distinctCount'})` |
|
|
119
|
+
| Ratio / composite metrics (conversion %, ARPU, blends) | `evaluateFormula` (`lib/verify/formula.js`) over emulator rows |
|
|
91
120
|
| Schema integrity / column coverage | DuckDB |
|
|
92
121
|
| Identity-model invariants | DuckDB |
|
|
93
122
|
| Experiment invariants | DuckDB |
|
|
94
|
-
|
|
|
123
|
+
| True bespokes (no emulator analysis fits) | DuckDB |
|
|
124
|
+
|
|
125
|
+
The emulator now covers lifecycle, flows, sessions, event breakdowns, formulas, and compounded retention — DuckDB's remit is schema / identity / experiment invariants plus true bespoke shapes. If a "bespoke" check is really a funnel, frequency, path, or breakdown in disguise, it belongs in the emulator.
|
|
95
126
|
|
|
96
127
|
**Hand-written DuckDB funnel SQL diverges from Mixpanel — never hand-roll.** If you find yourself writing `WITH step1 AS ..., step2 AS ...` for a funnel, STOP — use `emulateBreakdown` with `funnelFrequency` instead.
|
|
97
128
|
|
|
@@ -107,18 +138,18 @@ For emulator details, identity-model dungeons (must pass `profiles`), and time-s
|
|
|
107
138
|
**Everything about a dungeon lives in its folder.** When the dungeon being
|
|
108
139
|
verified is a user dungeon at `dungeons/user/<name>/<name>.js`, write ALL
|
|
109
140
|
generated artifacts into `dungeons/user/<name>/`:
|
|
110
|
-
- `hook-results.md` (Step
|
|
111
|
-
- `hook-query-log.txt` (Step
|
|
112
|
-
- `<name>-verifications.sql` (Step
|
|
141
|
+
- `hook-results.md` (Step 7)
|
|
142
|
+
- `hook-query-log.txt` (Step 6)
|
|
143
|
+
- `<name>-verifications.sql` (Step 7b)
|
|
113
144
|
|
|
114
145
|
The ONLY exception is the throwaway verification data the run writes to
|
|
115
146
|
`./data/` (`verify-*` event/user files) — that stays in `./data/` and is
|
|
116
|
-
deleted in Step
|
|
147
|
+
deleted in Step 8.
|
|
117
148
|
|
|
118
149
|
For non-user dungeons (technical/vertical) or batch runs across many dungeons,
|
|
119
150
|
fall back to `./research/` for `hook-results.md` / `hook-query-log.txt`.
|
|
120
151
|
|
|
121
|
-
### Step
|
|
152
|
+
### Step 6: Stash query log
|
|
122
153
|
|
|
123
154
|
Write every DuckDB query execution to `hook-query-log.txt`:
|
|
124
155
|
- **User dungeon:** always write to `dungeons/user/<name>/hook-query-log.txt`.
|
|
@@ -126,21 +157,21 @@ Write every DuckDB query execution to `hook-query-log.txt`:
|
|
|
126
157
|
|
|
127
158
|
Format and conventions: see [report-format.md "Query log format"](references/report-format.md#query-log-format).
|
|
128
159
|
|
|
129
|
-
### Step
|
|
160
|
+
### Step 7: Write `hook-results.md`
|
|
130
161
|
|
|
131
|
-
Write to `dungeons/user/<name>/hook-results.md` for a user dungeon, else `./research/hook-results.md`. Use the templates in [report-format.md](references/report-format.md):
|
|
162
|
+
Write to `dungeons/user/<name>/hook-results.md` for a user dungeon, else `./research/hook-results.md`. For story-backed dungeons, **the report renders the runner's JSON** (`verify-stories.mjs --json`): the hook summary table comes straight from the runner's per-story verdicts (story id, hook, archetype, observed vs target, verdict), and detailed blocks exist only for the Step-3 investigation targets. Use the templates in [report-format.md](references/report-format.md):
|
|
132
163
|
- Single-dungeon report structure
|
|
133
164
|
- Multi-dungeon report structure (when batch mode)
|
|
134
165
|
- Per-hook detail block
|
|
135
|
-
- Verdict criteria (5-tier)
|
|
166
|
+
- Verdict criteria (5-tier, mechanical)
|
|
136
167
|
|
|
137
168
|
**Order failures first** within each dungeon section: INVERSE → NONE → WEAK → STRONG → NAILED. Sort the summary table the same way. Actionable issues at the top.
|
|
138
169
|
|
|
139
|
-
### Step
|
|
170
|
+
### Step 7b: Write verification SQL (mandatory for user dungeons)
|
|
140
171
|
|
|
141
|
-
When verifying a dungeon in `dungeons/user/`, also write a standalone DuckDB SQL file alongside the dungeon in its folder at `dungeons/user/<name>/<name>-verifications.sql`. Vertical dungeons already have their SQL
|
|
172
|
+
When verifying a dungeon in `dungeons/user/`, also write a standalone DuckDB SQL file alongside the dungeon in its folder at `dungeons/user/<name>/<name>-verifications.sql`. Vertical dungeons already have their SQL co-located at `dungeons/vertical/<name>/<name>.sql`. Format: see [report-format.md "Verification SQL file"](references/report-format.md#verification-sql-file-mandatory-for-user-dungeons).
|
|
142
173
|
|
|
143
|
-
### Step
|
|
174
|
+
### Step 8: Cleanup
|
|
144
175
|
|
|
145
176
|
```bash
|
|
146
177
|
rm -f ./data/verify-* ./verify-*
|
|
@@ -171,7 +202,7 @@ Tell the user:
|
|
|
171
202
|
1. Report path: `dungeons/user/<name>/hook-results.md` (user dungeon) or `./research/hook-results.md`
|
|
172
203
|
2. Verification SQL path (for user dungeons): `dungeons/user/<name>/<name>-verifications.sql`
|
|
173
204
|
3. Query log path (if written): alongside the report (`dungeons/user/<name>/hook-query-log.txt`, else `./research/hook-query-log.txt`)
|
|
174
|
-
4.
|
|
205
|
+
4. Verdict counts from the story runner (per dungeon if batch mode), plus which stories needed LLM investigation
|
|
175
206
|
5. One-line summary of the most interesting finding
|
|
176
207
|
|
|
177
208
|
If hooks failed, note that `hook-results.md` can be used as context for fixing them: "read hook-results.md and fix the failing hooks in <dungeon-file>".
|
|
@@ -14,13 +14,10 @@ Mixpanel does NOT count the way naive SQL does. The verifier (and any DuckDB que
|
|
|
14
14
|
| Sessions | 3-trigger split: timeout `>`, max duration `>`, day-idx change (`session_query.cpp:906-911`) | Trust pre-stamped `session_id`; group by `(user, session_id)` |
|
|
15
15
|
| Retention | Birth-anchored, ms-strict gate (default `birth_can_retain=false` → `<`), bucketed by `floor((ret−birth)/unit)` (`retention_query.cpp:1097-1109,1228-1231`) | Use `emulateBreakdown` with `retention` |
|
|
16
16
|
|
|
17
|
-
**Known divergences from Mixpanel C++** (1.
|
|
17
|
+
**Known divergences from Mixpanel C++** (1.6.0):
|
|
18
18
|
- `countDistinctPeriods` default = `algorithm: 'calendar'` (UTC bucket).
|
|
19
19
|
Mixpanel C++ (`addiction_query.cpp:359`) uses ROLLING window. Pass
|
|
20
20
|
`algorithm: 'rolling'` for exact Frequency-Distribution parity.
|
|
21
|
-
- COMPOUNDED retention is NOT implemented — verifier silently ignores
|
|
22
|
-
`compounded: true`. Use DuckDB or query Mixpanel directly for "DAU
|
|
23
|
-
coming back" reports.
|
|
24
21
|
- Touchpoint sampling: generator stamps uniform-random across user
|
|
25
22
|
lifetime; verifier reads last-N before conversion (matches C++).
|
|
26
23
|
For users with ≤10 attribution events lifetime, no divergence.
|
|
@@ -31,7 +28,7 @@ Full rules: see [HOOKS.md Section 2](../../../../HOOKS.md#2-how-mixpanel-counts-
|
|
|
31
28
|
|
|
32
29
|
## When to use the emulator vs DuckDB
|
|
33
30
|
|
|
34
|
-
The emulator (`emulateBreakdown` from `@ak--47/dungeon-master/verify`) implements Mixpanel's rules natively. **ALWAYS use the emulator for funnel, frequency, aggregate, TTC, and
|
|
31
|
+
The emulator (`emulateBreakdown` from `@ak--47/dungeon-master/verify`) implements Mixpanel's rules natively. **ALWAYS use the emulator for funnel, frequency, aggregate, TTC, attribution, retention, lifecycle, flows, sessions, and event-breakdown patterns.** Hand-written DuckDB queries for these pattern types diverge from what Mixpanel shows in reports — even when they look correct.
|
|
35
32
|
|
|
36
33
|
Use DuckDB ONLY for:
|
|
37
34
|
- Schema integrity checks (column coverage, flag detection)
|
|
@@ -51,10 +48,33 @@ If you find yourself writing `WITH step1 AS ..., step2 AS ...` for a funnel, STO
|
|
|
51
48
|
| Funnel TTC by user property | `timeToConvert` | "Trial users take 4× longer than enterprise" |
|
|
52
49
|
| First/last touch attribution | `attributedBy` | "Conversions by Source" |
|
|
53
50
|
| Birth retention curves | `retention` | "Sign Up → Login on day N" — requires `cohortEvent`, `returnEvent`, `dayBuckets` |
|
|
51
|
+
| Compounded retention ("DAU coming back") | `retention` + `compounded: true` | Sets `returnEvent := cohortEvent` internally; throws on a conflicting `returnEvent` |
|
|
54
52
|
| Per-session metrics | `sessionMetrics` | Count / duration / events distributions per session |
|
|
53
|
+
| Lifecycle states (new / retained / resurrected / dormant) | `lifecycle` | "Resurrection wave in week N" — per-period user-state classification |
|
|
54
|
+
| Flows / top paths | `topPaths` | "What do users do after X" — Sankey top-path shares |
|
|
55
|
+
| Event totals segmented by property | `eventBreakdown` | "Purchases by plan tier" — `countType: 'general' \| 'sessions'`; list props fan out per item |
|
|
56
|
+
| Uniques per segment | `uniques` | "Unique users by plan tier" |
|
|
57
|
+
| Distinct property values | `distinctCount` | "How many distinct SKUs" — see below |
|
|
58
|
+
| Ratio / composite metrics | `evaluateFormula` (`lib/verify/formula.js`) | Conversion %, ARPU, any PEMDAS formula over emulator series |
|
|
55
59
|
|
|
56
60
|
Cross-cutting on EVERY type: `timeBucket: 'day' | 'week' | 'month'` partitions events into UTC buckets and emits one row per period.
|
|
57
61
|
|
|
62
|
+
### `distinctCount`
|
|
63
|
+
|
|
64
|
+
Counts distinct values of a flat event property, Mixpanel COUNT_DISTINCT-style, with a top-values breakdown:
|
|
65
|
+
|
|
66
|
+
```js
|
|
67
|
+
emulateBreakdown(events, {
|
|
68
|
+
type: 'distinctCount',
|
|
69
|
+
property: 'sku', // required — flat property name (dot-paths not supported)
|
|
70
|
+
event: 'Purchase', // optional — restrict to one event type; omit = all events
|
|
71
|
+
topN: 25, // optional — top values by count (default 25)
|
|
72
|
+
});
|
|
73
|
+
// → [{ distinct_count: 143, top_values: [{ value: 'SKU-001', count: 812 }, ...] }]
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Single-row result. Use it for catalog-breadth stories ("power sellers list 5× more distinct SKUs") where `uniques` / `eventBreakdown` answer a different question (users or totals per bucket, not distinct values of the property itself).
|
|
77
|
+
|
|
58
78
|
Quick emulator script:
|
|
59
79
|
|
|
60
80
|
```js
|
|
@@ -98,7 +118,7 @@ For CI-style assertions, use `verifyDungeon` with a checks array; see `tests/e2e
|
|
|
98
118
|
| `readFileSync` ENOMEM on shards >500MB | Node string cap at ~512MB | Stream-load with `readline.createInterface` over `data/PREFIX-EVENTS*.json` glob |
|
|
99
119
|
| Hook reads `e.event === 'login'` but cohort empty | `login` is a funnel-step event auto-promoted to `isStrictEvent: true` | Add `isStrictEvent: false` to the event config to keep standalone occurrences |
|
|
100
120
|
|
|
101
|
-
When writing per-dungeon verify scripts, follow the template in HOOKS.md §9.9. Reference proofs for all
|
|
121
|
+
When writing per-dungeon verify scripts, follow the template in HOOKS.md §9.9. Reference proofs for all 22 vertical dungeons live at `dungeons/vertical/<name>/` — consult them as exemplars before authoring a new one.
|
|
102
122
|
|
|
103
123
|
**Coverage discipline:** count documented hooks in the dungeon's top-level comment block; count `check()` calls in your verify script; the two MUST match. A "NAILED 7/7" claim against an 11-hook dungeon is misleading — either add the missing checks or document the limitation explicitly in the verify script and status file.
|
|
104
124
|
|