@ak--47/dungeon-master 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +96 -0
- package/.claude/skills/create-project/provision.mjs +336 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +211 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +3 -2
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -0,0 +1,1480 @@
|
|
|
1
|
+
// ── IMPORTS ──
|
|
2
|
+
import dayjs from "dayjs";
|
|
3
|
+
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
5
|
+
import { uid } from "ak-tools";
|
|
6
|
+
import { weighNumRange, integer, weighChoices, decimal, initChance } from "@ak--47/dungeon-master/utils";
|
|
7
|
+
import { scaleFunnelTTC } from "@ak--47/dungeon-master/hook-helpers";
|
|
8
|
+
/** @typedef {import("../../../types").Dungeon} Config */
|
|
9
|
+
|
|
10
|
+
// ── OVERVIEW ──
|
|
11
|
+
/*
|
|
12
|
+
* NAME: ShopStream
|
|
13
|
+
* APP: eCommerce marketplace with integrated video content. Users browse
|
|
14
|
+
* products, watch videos, build carts, and check out. A signup
|
|
15
|
+
* funnel converts anonymous browsers into registered users; video
|
|
16
|
+
* engagement (like/dislike) runs alongside the shopping flow.
|
|
17
|
+
* SCALE: 42,000 users, ~2M events, 121 days (2026-01-01 → 2026-05-01)
|
|
18
|
+
* CORE LOOP: page view → view item → save/add to cart → checkout (+ video like/dislike side loop)
|
|
19
|
+
*
|
|
20
|
+
* EVENTS (9):
|
|
21
|
+
* page view (10) = save item (10) > view item (8) > watch video (8) > like video (6)
|
|
22
|
+
* > add to cart (4) > dislike video (4) > checkout (2) > sign up (1)
|
|
23
|
+
*
|
|
24
|
+
* FUNNELS (4):
|
|
25
|
+
* - Signup Flow: page view → view item → save item → page view → sign up (50%)
|
|
26
|
+
* - Video Likes: watch video → like video → watch video → like video (60%)
|
|
27
|
+
* - Video Dislikes: watch video → dislike video → watch video → dislike video (20%)
|
|
28
|
+
* - eCommerce Purchase: view item → view item → add to cart → view item → add to cart → checkout (15%, A/B/C experiment)
|
|
29
|
+
*
|
|
30
|
+
* USER PROPS: title, luckyNumber, spiritAnimal, theme
|
|
31
|
+
* SUPER PROPS: theme
|
|
32
|
+
* SCD PROPS: loyalty_tier (bronze/silver/gold/platinum, monthly fuzzy, max 8)
|
|
33
|
+
* GROUPS: none
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
// ── HOOK STORIES ──
|
|
37
|
+
/*
|
|
38
|
+
* ----------------------------------------------------------------------------
|
|
39
|
+
* Hook 1: Signup Flow Improvement (event + everything)
|
|
40
|
+
* ----------------------------------------------------------------------------
|
|
41
|
+
*
|
|
42
|
+
* PATTERN: 7 days ago, signup_flow switches from "v1" to "v2". Before
|
|
43
|
+
* that date, 50% of sign ups are tagged _drop and removed in the
|
|
44
|
+
* everything hook, simulating a broken flow that got fixed.
|
|
45
|
+
*
|
|
46
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
47
|
+
*
|
|
48
|
+
* Report 1: Sign Ups by Flow Version
|
|
49
|
+
* - Report type: Insights
|
|
50
|
+
* - Event: "sign up"
|
|
51
|
+
* - Measure: Total
|
|
52
|
+
* - Breakdown: "signup_flow"
|
|
53
|
+
* - Line chart by day
|
|
54
|
+
* - Expected: v1 disappears at day -7 with ZERO v2 before the fix and zero
|
|
55
|
+
* v1 after (tag purity is exact — the hook branches on the boundary)
|
|
56
|
+
*
|
|
57
|
+
* Report 2: Pre vs Post Volume
|
|
58
|
+
* - Report type: Insights
|
|
59
|
+
* - Event: "sign up"
|
|
60
|
+
* - Measure: Total, daily average pre vs post fix date
|
|
61
|
+
* - Expected: post-fix daily rate ~5-7x the pre-fix average. The hook
|
|
62
|
+
* contributes exactly 2x (50% of pre-fix signups dropped); the rest is
|
|
63
|
+
* the acquisition ramp — births skew heavily late in the window
|
|
64
|
+
* (measured 6.9x at iteration scale), so the jump reads as "fix landed
|
|
65
|
+
* during a growth phase"
|
|
66
|
+
*
|
|
67
|
+
* REAL-WORLD ANALOGUE: A broken signup flow silently halved conversions
|
|
68
|
+
* until a release shipped a fix; daily signups roughly doubled overnight.
|
|
69
|
+
*
|
|
70
|
+
* ----------------------------------------------------------------------------
|
|
71
|
+
* Hook 2: Watch Time Inflection (event)
|
|
72
|
+
* ----------------------------------------------------------------------------
|
|
73
|
+
*
|
|
74
|
+
* PATTERN: 30 days ago, watch time shifts. Before that date, watchTimeSec
|
|
75
|
+
* is reduced by 25-79%. After, it is increased by 25-79%. Creates a
|
|
76
|
+
* clear before/after inflection.
|
|
77
|
+
*
|
|
78
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
79
|
+
*
|
|
80
|
+
* Report 1: Avg Watch Time Over Time
|
|
81
|
+
* - Report type: Insights
|
|
82
|
+
* - Event: "watch video"
|
|
83
|
+
* - Measure: Average of "watchTimeSec"
|
|
84
|
+
* - Line chart by day
|
|
85
|
+
* - Expected: clear upward inflection ~ 30 days ago
|
|
86
|
+
*
|
|
87
|
+
* Report 2: Watch Time Distribution Pre vs Post
|
|
88
|
+
* - Report type: Insights
|
|
89
|
+
* - Event: "watch video"
|
|
90
|
+
* - Measure: Average of "watchTimeSec"
|
|
91
|
+
* - Compare date ranges (last 30 days vs prior 91 days)
|
|
92
|
+
* - Expected: post/pre avg ratio = E[1+f]/E[1-f] with f uniform on
|
|
93
|
+
* [0.25, 0.79] → 1.52/0.48 ≈ 3.17 (measured 3.19 at iteration scale)
|
|
94
|
+
*
|
|
95
|
+
* REAL-WORLD ANALOGUE: An algorithm or UX change (autoplay, recs)
|
|
96
|
+
* inflects average watch duration sharply on a release date.
|
|
97
|
+
*
|
|
98
|
+
* ----------------------------------------------------------------------------
|
|
99
|
+
* Hook 3: Toys + Shoes Cart Correlation (event)
|
|
100
|
+
* ----------------------------------------------------------------------------
|
|
101
|
+
*
|
|
102
|
+
* PATTERN: Checkout carts with toys get a shoes item injected (and vice
|
|
103
|
+
* versa). Carts with neither toys nor shoes have all item prices
|
|
104
|
+
* discounted to 75-90% of normal.
|
|
105
|
+
*
|
|
106
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
107
|
+
*
|
|
108
|
+
* Report 1: Cart Category Co-occurrence
|
|
109
|
+
* - Report type: Insights
|
|
110
|
+
* - Event: "checkout"
|
|
111
|
+
* - Measure: Total
|
|
112
|
+
* - Breakdown: "category" (flattened item category)
|
|
113
|
+
* - Expected: P(shoes | toys in cart) ≈ 0.64 predicted (baseline ~0.29
|
|
114
|
+
* organic + injection succeeding ~49% of the time — the injected item
|
|
115
|
+
* comes from a random 1-20-item cart that must contain a shoe), vs
|
|
116
|
+
* P(shoes | no toys) ≈ 0.11 — reciprocal toy-injection drains
|
|
117
|
+
* shoes-without-toys carts. Measured 0.71 vs 0.11 → ~6.4x lift
|
|
118
|
+
*
|
|
119
|
+
* Report 2: Cart Value by Category Mix
|
|
120
|
+
* - Report type: Insights
|
|
121
|
+
* - Event: "checkout"
|
|
122
|
+
* - Measure: Average of "amount"
|
|
123
|
+
* - Breakdown: "category"
|
|
124
|
+
* - Expected: neither-cart avg item amount = mean(uniform 0.75-0.9)
|
|
125
|
+
* ≈ 0.825x carts holding toys or shoes (measured 0.824)
|
|
126
|
+
*
|
|
127
|
+
* REAL-WORLD ANALOGUE: Family shoppers buying for kids tend to bundle
|
|
128
|
+
* toys with shoes; a market-basket pattern that retailers exploit.
|
|
129
|
+
*
|
|
130
|
+
* ----------------------------------------------------------------------------
|
|
131
|
+
* Hook 4: Quality to Watch Time Correlation (event)
|
|
132
|
+
* ----------------------------------------------------------------------------
|
|
133
|
+
*
|
|
134
|
+
* PATTERN: Video quality multiplies watchTimeSec: 2160p=1.5x, 1440p=1.4x,
|
|
135
|
+
* 1080p=1.3x, 720p=1.15x, 480p=1.0x, 360p=0.85x, 240p=0.7x.
|
|
136
|
+
*
|
|
137
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
138
|
+
*
|
|
139
|
+
* Report 1: Avg Watch Time by Quality
|
|
140
|
+
* - Report type: Insights
|
|
141
|
+
* - Event: "watch video"
|
|
142
|
+
* - Measure: Average of "watchTimeSec"
|
|
143
|
+
* - Breakdown: "quality"
|
|
144
|
+
* - Expected: strictly monotonic increase from 240p to 2160p;
|
|
145
|
+
* 2160p/240p avg ratio = 1.5/0.7 ≈ 2.14 (measured 2.07 — H2's
|
|
146
|
+
* inflection factor composes multiplicatively but is quality-blind,
|
|
147
|
+
* so it cancels in the ratio)
|
|
148
|
+
*
|
|
149
|
+
* Report 2: Quality Distribution
|
|
150
|
+
* - Report type: Insights
|
|
151
|
+
* - Event: "watch video"
|
|
152
|
+
* - Measure: Total
|
|
153
|
+
* - Breakdown: "quality"
|
|
154
|
+
* - Expected: viewers split across all quality tiers; HD tiers earn more time
|
|
155
|
+
*
|
|
156
|
+
* REAL-WORLD ANALOGUE: Higher-resolution streams correlate with longer
|
|
157
|
+
* sessions because they signal a better connection and more invested viewer.
|
|
158
|
+
*
|
|
159
|
+
* ----------------------------------------------------------------------------
|
|
160
|
+
* Hook 5: Item Flattening (event)
|
|
161
|
+
* ----------------------------------------------------------------------------
|
|
162
|
+
*
|
|
163
|
+
* PATTERN: Events with an item array property (view item, add to cart,
|
|
164
|
+
* save item) get the first item's fields (category, amount, slug, etc.)
|
|
165
|
+
* flattened onto the event record as top-level properties, and the nested
|
|
166
|
+
* "item" array is deleted. Checkout's "cart" array stays nested. The
|
|
167
|
+
* discriminating signal vs the schema defaults: without the hook, every
|
|
168
|
+
* item-event would carry slug="item" and the placeholder assetPreview;
|
|
169
|
+
* after flattening, slug is always the "<descriptor>-<suffix>" compound
|
|
170
|
+
* of the actual product (100% of item-events, no "item" column left).
|
|
171
|
+
*
|
|
172
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
173
|
+
*
|
|
174
|
+
* Report 1: View Items by Category
|
|
175
|
+
* - Report type: Insights
|
|
176
|
+
* - Event: "view item"
|
|
177
|
+
* - Measure: Total
|
|
178
|
+
* - Breakdown: "category"
|
|
179
|
+
* - Expected: clean breakdown across category values without nested-property issues
|
|
180
|
+
*
|
|
181
|
+
* Report 2: Add to Cart Avg Amount by Category
|
|
182
|
+
* - Report type: Insights
|
|
183
|
+
* - Event: "add to cart"
|
|
184
|
+
* - Measure: Average of "amount"
|
|
185
|
+
* - Breakdown: "category"
|
|
186
|
+
* - Expected: per-category averages render correctly off flat properties
|
|
187
|
+
*
|
|
188
|
+
* REAL-WORLD ANALOGUE: Analytics teams routinely flatten nested cart
|
|
189
|
+
* objects onto events so downstream tools can group/filter without joins.
|
|
190
|
+
*
|
|
191
|
+
* ----------------------------------------------------------------------------
|
|
192
|
+
* Hook 6: View-Item Magic Number (everything)
|
|
193
|
+
* ----------------------------------------------------------------------------
|
|
194
|
+
*
|
|
195
|
+
* PATTERN: Users who view 3-8 items in the dataset window are in the
|
|
196
|
+
* "considered buyer" sweet spot — every cart item amount and total_value
|
|
197
|
+
* gets boosted ~25%, AND ~45% of their add-to-cart events are cloned
|
|
198
|
+
* (with time offsets) to elevate their cart add rate. Users who view
|
|
199
|
+
* 9 or more items are over-engaged window-shoppers; ~30% of their
|
|
200
|
+
* checkout events are dropped (decision paralysis / browse without buy).
|
|
201
|
+
* No flag is stamped — discoverable only by binning users on view-item
|
|
202
|
+
* COUNT and comparing avg cart item amount or add-to-cart rate.
|
|
203
|
+
*
|
|
204
|
+
* MEASUREMENT CAVEAT: view-item count is dominated by eCommerce Purchase
|
|
205
|
+
* funnel passes (3 views per pass, weight 10), so the "over" bin holds
|
|
206
|
+
* ~89% of users and is mechanically funnel-heavy — raw checkouts-per-user
|
|
207
|
+
* INCREASES with view count despite the drop. The clean signals are
|
|
208
|
+
* per-item and per-ratio, not per-user counts:
|
|
209
|
+
* - sweet/over avg cart item amount ≈ 1.25x (the boost, exact knob;
|
|
210
|
+
* measured 1.248 — H3's discount hits both bins equally and cancels)
|
|
211
|
+
* - sweet/over add-to-carts-per-view-item ≈ 1.37x (the 1.45x clone
|
|
212
|
+
* inflation punching through the over bin's funnel-heavier cart mix)
|
|
213
|
+
* - over/sweet checkouts-per-add-to-cart ≈ 0.54 (composite: the 30%
|
|
214
|
+
* drop, plus failed funnel attempts leaving cart-without-checkout
|
|
215
|
+
* prefixes disproportionately in the over bin)
|
|
216
|
+
*
|
|
217
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
218
|
+
*
|
|
219
|
+
* Report 1: Avg Cart Item Amount by View-Item Bucket
|
|
220
|
+
* - Report type: Insights (with cohort)
|
|
221
|
+
* - Cohort A: users with 3-8 "view item" events
|
|
222
|
+
* - Cohort C: users with >= 9 "view item" events
|
|
223
|
+
* - Event: "checkout"
|
|
224
|
+
* - Measure: Average of cart item "amount"
|
|
225
|
+
* - Compare cohort A vs cohort C
|
|
226
|
+
* - Expected: cohort A ~ 1.25x higher cart item amounts than C
|
|
227
|
+
*
|
|
228
|
+
* Report 2: Add-to-Cart Rate by Browse Intensity
|
|
229
|
+
* - Report type: Insights (with cohort)
|
|
230
|
+
* - Cohorts A (3-8 views) vs C (9+ views)
|
|
231
|
+
* - Event: "add to cart" normalized by "view item"
|
|
232
|
+
* - Expected: cohort A ~ 1.37x adds-per-view; checkouts-per-add for C
|
|
233
|
+
* ~ 0.54x of A
|
|
234
|
+
*
|
|
235
|
+
* REAL-WORLD ANALOGUE: A focused buyer who reviews a handful of items
|
|
236
|
+
* tends to convert at higher cart value; an excessive browser is a
|
|
237
|
+
* tire-kicker who abandons more often.
|
|
238
|
+
*
|
|
239
|
+
* ----------------------------------------------------------------------------
|
|
240
|
+
* Hook 7: Signup Flow Time-to-Convert (everything)
|
|
241
|
+
* ----------------------------------------------------------------------------
|
|
242
|
+
*
|
|
243
|
+
* PATTERN: The Signup Flow funnel (page view → view item → save item →
|
|
244
|
+
* page view → sign up) has its time-to-convert scaled by the user's
|
|
245
|
+
* loyalty tier from SCD data (meta.scd.loyalty_tier). Gold and platinum
|
|
246
|
+
* users complete the funnel 0.67x faster; bronze users take 1.33x longer;
|
|
247
|
+
* silver is unaffected. The hook reads the latest SCD entry for
|
|
248
|
+
* loyalty_tier, falling back to a deterministic hash of user_id for
|
|
249
|
+
* consistent tier assignment. It anchors on the first "sign up" event,
|
|
250
|
+
* looks back up to 2 days for all funnel-step events in that window,
|
|
251
|
+
* and scales the whole cluster via scaleFunnelTTC.
|
|
252
|
+
*
|
|
253
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
254
|
+
*
|
|
255
|
+
* Report 1: Signup Funnel TTC by Loyalty Tier
|
|
256
|
+
* - Report type: Funnels
|
|
257
|
+
* - Steps: "page view" → "view item" → "save item" → "page view" → "sign up"
|
|
258
|
+
* - Measure: Median time to convert
|
|
259
|
+
* - Breakdown: "loyalty_tier" (SCD property)
|
|
260
|
+
* - Expected: gold/platinum ~ 0.67x median TTC vs silver; bronze ~ 1.33x.
|
|
261
|
+
* bronze/gold ratio ceiling = 1.33/0.67 ≈ 1.99; measured 1.85 —
|
|
262
|
+
* organic (non-funnel) events inside the 2-day lookback window get
|
|
263
|
+
* scaled too, and pre-auth funnel steps carry device_id only, so
|
|
264
|
+
* identity must be resolved through the profile's device pool
|
|
265
|
+
*
|
|
266
|
+
* Report 2: Signup Funnel Conversion by Loyalty Tier
|
|
267
|
+
* - Report type: Funnels
|
|
268
|
+
* - Steps: same as above
|
|
269
|
+
* - Breakdown: "loyalty_tier"
|
|
270
|
+
* - Expected: conversion rate similar across tiers (TTC changes, not conversion)
|
|
271
|
+
*
|
|
272
|
+
* REAL-WORLD ANALOGUE: Loyal, high-tier customers already trust the
|
|
273
|
+
* platform and breeze through signup flows, while new or low-engagement
|
|
274
|
+
* users deliberate longer before committing.
|
|
275
|
+
*
|
|
276
|
+
* ----------------------------------------------------------------------------
|
|
277
|
+
* Hook 8: Checkout Flow Experiment (funnel experiment config)
|
|
278
|
+
* ----------------------------------------------------------------------------
|
|
279
|
+
*
|
|
280
|
+
* PATTERN: A/B/C experiment on the eCommerce Purchase funnel. Control
|
|
281
|
+
* group uses base conversion, "Express Checkout" variant gets 1.25x
|
|
282
|
+
* conversion multiplier, "Social Proof" gets 1.15x conversion and 0.9x
|
|
283
|
+
* time-to-convert. Starts 30 days before dataset end.
|
|
284
|
+
*
|
|
285
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
286
|
+
*
|
|
287
|
+
* Report 1: Experiment Events by Variant
|
|
288
|
+
* - Report type: Insights
|
|
289
|
+
* - Event: "$experiment_started"
|
|
290
|
+
* - Measure: Total
|
|
291
|
+
* - Breakdown: "$experiment_variant"
|
|
292
|
+
* - Expected: roughly equal user split (deterministic hash thirds).
|
|
293
|
+
* "Variant name" / "Experiment name" are stamped ONLY on
|
|
294
|
+
* $experiment_started, not on the funnel step events
|
|
295
|
+
*
|
|
296
|
+
* Report 2: Purchase Funnel Conversion by Variant
|
|
297
|
+
* - Report type: Funnels
|
|
298
|
+
* - Funnel: view item → add to cart → checkout
|
|
299
|
+
* - Breakdown: "$experiment_variant"
|
|
300
|
+
* - Expected: Express Checkout > Social Proof > Control. Effective
|
|
301
|
+
* per-attempt rates compose with H9's theme multiplier (engine applies
|
|
302
|
+
* experiment first, funnel-pre after): Control ≈ 16.2%, Express ≈
|
|
303
|
+
* 20.2%, Social ≈ 17.8% theme-weighted → Express/Control ≈ 1.25,
|
|
304
|
+
* Social/Control ≈ 1.10. Social Proof also converts 0.9x faster
|
|
305
|
+
* (funnel timeToConvert defaults to 1h)
|
|
306
|
+
* - MEASUREMENT CAVEAT: checkout is also a weight-2 organic event that
|
|
307
|
+
* clusters in the same session as the funnel pass (~48K organic vs
|
|
308
|
+
* ~2K funnel checkouts). Naive "checkout within 75min" pairing adds a
|
|
309
|
+
* variant-independent floor that compresses the observable ratio to
|
|
310
|
+
* ~1.06 and mean TTC from ~60min to ~34min. Verification pairs
|
|
311
|
+
* STRICTLY — checkout in-window AND >= 5 view/add steps between
|
|
312
|
+
* $experiment_started and checkout (a converted pass always emits its
|
|
313
|
+
* 5 steps; failed passes emit 1-5). Mixpanel's Funnels report handles
|
|
314
|
+
* this natively via ordered-sequence matching, so the report ratios
|
|
315
|
+
* land near the composed rates above. Engine effect verified by an
|
|
316
|
+
* isolated no-hook repro (Express lift present; Social/Control TTC
|
|
317
|
+
* 0.888 ≈ knob 0.9)
|
|
318
|
+
*
|
|
319
|
+
* REAL-WORLD ANALOGUE: Product team tests a streamlined checkout and a
|
|
320
|
+
* social proof variant against the existing flow to lift conversion.
|
|
321
|
+
*
|
|
322
|
+
* ----------------------------------------------------------------------------
|
|
323
|
+
* Hook 9: Dark Theme Power Users (funnel-pre)
|
|
324
|
+
* ----------------------------------------------------------------------------
|
|
325
|
+
*
|
|
326
|
+
* PATTERN: Dark-theme users convert 1.3x better on the purchase funnel;
|
|
327
|
+
* light-theme users convert at 0.85x. Custom theme is unaffected.
|
|
328
|
+
*
|
|
329
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
330
|
+
*
|
|
331
|
+
* Report 1: Purchase Funnel by Theme
|
|
332
|
+
* - Report type: Funnels
|
|
333
|
+
* - Funnel: view item → add to cart → checkout
|
|
334
|
+
* - Breakdown: "theme"
|
|
335
|
+
* - Expected: dark > custom > light conversion rate. Effective funnel
|
|
336
|
+
* rates (base 15): dark round(15*1.3)=20, light round(15*0.85)=13,
|
|
337
|
+
* custom 15 → dark/light per-attempt ratio ≈ 1.54
|
|
338
|
+
*
|
|
339
|
+
* Report 2: Checkout Count by Theme
|
|
340
|
+
* - Report type: Insights
|
|
341
|
+
* - Event: "checkout"
|
|
342
|
+
* - Measure: Total per user
|
|
343
|
+
* - Breakdown: "theme"
|
|
344
|
+
* - Expected: dark/light checkouts-per-user ≈ 1.48 (the 1.54 funnel
|
|
345
|
+
* ratio diluted by theme-blind organic checkouts, weight 2);
|
|
346
|
+
* custom/light ≈ 1.13
|
|
347
|
+
*
|
|
348
|
+
* REAL-WORLD ANALOGUE: Power users who customize their UI (dark mode)
|
|
349
|
+
* tend to be more committed and convert at higher rates.
|
|
350
|
+
*
|
|
351
|
+
* ----------------------------------------------------------------------------
|
|
352
|
+
* Hook 10: Save-Item Retention (everything — retention magic number)
|
|
353
|
+
* ----------------------------------------------------------------------------
|
|
354
|
+
*
|
|
355
|
+
* PATTERN: Born-in-dataset users who perform 2+ "save item" events in
|
|
356
|
+
* their first 10 days retain long-term. Users below that threshold lose
|
|
357
|
+
* ~70% of events after day 25, simulating churn. No flag is stamped —
|
|
358
|
+
* discoverable only by segmenting users on early save-item count and
|
|
359
|
+
* comparing retention or late-period activity. NOTE: the Signup Flow
|
|
360
|
+
* funnel includes a "save item" step, so every signer has exactly one
|
|
361
|
+
* guaranteed save — the threshold is really "made a 2nd, organic save".
|
|
362
|
+
*
|
|
363
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
364
|
+
*
|
|
365
|
+
* Report 1: Retention by Early Save Count
|
|
366
|
+
* - Report type: Retention
|
|
367
|
+
* - First event: any event
|
|
368
|
+
* - Return event: any event
|
|
369
|
+
* - Cohort A: users with >= 2 "save item" in first 10 days
|
|
370
|
+
* - Cohort B: users with < 2 "save item" in first 10 days
|
|
371
|
+
* - Expected: Cohort A retains at 4+ weeks; Cohort B drops sharply after
|
|
372
|
+
* week 3. Applies to BORN-IN users only (~4% of users sign up in the
|
|
373
|
+
* window; savers are ~15-20% of eligible born users — a small,
|
|
374
|
+
* engineered cohort by design)
|
|
375
|
+
*
|
|
376
|
+
* Report 2: Post-Day-25 Activity
|
|
377
|
+
* - Report type: Insights
|
|
378
|
+
* - Event: any event
|
|
379
|
+
* - Filter: time > first_event + 25 days
|
|
380
|
+
* - Cohort A: >= 2 early saves; Cohort B: < 2 early saves
|
|
381
|
+
* - Expected: (B post/pre rate) / (A post/pre rate) ≈ 0.30 — the drop
|
|
382
|
+
* knob exactly, since the ratio-of-ratios cancels window lengths and
|
|
383
|
+
* the acquisition ramp (measured 0.24 at iteration scale)
|
|
384
|
+
*
|
|
385
|
+
* REAL-WORLD ANALOGUE: Saving items signals purchase intent and product
|
|
386
|
+
* engagement; users who curate a wishlist early are more likely to return.
|
|
387
|
+
*
|
|
388
|
+
* ============================================================================
|
|
389
|
+
* EXPECTED METRICS SUMMARY
|
|
390
|
+
* ============================================================================
|
|
391
|
+
*
|
|
392
|
+
* Hook | Metric | Derivation | Expected | Measured (full)
|
|
393
|
+
* -----|------------------------------------------|---------------------|----------|----------------
|
|
394
|
+
* H1 | v2-pre-fix + v1-post-fix count | boundary branch | 0 + 0 | 0 + 0
|
|
395
|
+
* H1 | daily signup rate post/pre | 2x drop x ramp | 5-9x | 8.25x
|
|
396
|
+
* H2 | avg watchTimeSec post/pre | 1.52/0.48 | 3.17x | 3.19x
|
|
397
|
+
* H3 | P(shoes|toys) / P(shoes|no toys) | injection composite | ~6x | 6.45x
|
|
398
|
+
* H3 | neither-cart avg amount vs either | mean(0.75..0.9) | 0.825x | 0.828x
|
|
399
|
+
* H4 | avg watchTimeSec 2160p/240p | 1.5/0.7 | 2.14x | 2.17x
|
|
400
|
+
* H5 | item-events flattened (no nested item) | deterministic | 100% | 100%
|
|
401
|
+
* H6 | sweet/over avg cart item amount | SWEET_CART_BOOST | 1.25x | 1.233x
|
|
402
|
+
* H6 | sweet/over add-to-carts per view | 1.45 clone, diluted | ~1.37x | 1.372x
|
|
403
|
+
* H6 | over/sweet checkouts per add-to-cart | 0.7 drop composite | ~0.54x | 0.521x
|
|
404
|
+
* H7 | signup TTC bronze/gold+plat median | 1.33/0.67 diluted | ~1.9x | 1.982x
|
|
405
|
+
* H8 | Express/Control strict-paired conversion | 20.2/16.2 composed | ~1.25x | 1.19x (1.07-1.53 across seeds)
|
|
406
|
+
* H8 | Social/Control strict-paired mean TTC | ttcMultiplier | ~0.9x | 0.902x
|
|
407
|
+
* H9 | dark/light checkouts per user | 20/13 diluted | ~1.48x | 1.412x
|
|
408
|
+
* H10 | (nonsaver post/pre) / (saver post/pre) | drop knob | ~0.30x | 0.289x
|
|
409
|
+
* ============================================================================
|
|
410
|
+
*/
|
|
411
|
+
|
|
412
|
+
// ── SCALE ──
|
|
413
|
+
const SEED = "simple is best";
|
|
414
|
+
const NUM_USERS = 42_000;
|
|
415
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
416
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
417
|
+
const EVENTS_PER_DAY = 0.37;
|
|
418
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
419
|
+
|
|
420
|
+
const chance = initChance(SEED);
|
|
421
|
+
|
|
422
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
423
|
+
const SIGNUP_FIX_DAYS_AGO = 7;
|
|
424
|
+
const WATCH_INFLECTION_DAYS_AGO = 30;
|
|
425
|
+
const WATCH_FACTOR_MIN = 0.25;
|
|
426
|
+
const WATCH_FACTOR_MAX = 0.79;
|
|
427
|
+
const VIEW_SWEET_MIN = 3;
|
|
428
|
+
const VIEW_SWEET_MAX = 8;
|
|
429
|
+
const VIEW_OVER_THRESHOLD = 9;
|
|
430
|
+
const SWEET_CART_BOOST = 1.25;
|
|
431
|
+
const SWEET_CLONE_LIKELIHOOD = 45;
|
|
432
|
+
const OVER_CHECKOUT_DROP_LIKELIHOOD = 30;
|
|
433
|
+
const LOYALTY_TTC_FAST = 0.67;
|
|
434
|
+
const LOYALTY_TTC_SLOW = 1.33;
|
|
435
|
+
const LOYALTY_LOOKBACK_DAYS = 2;
|
|
436
|
+
// MIN must be 2: the Signup Flow funnel includes a "save item" step, so every
|
|
437
|
+
// signer carries exactly one funnel-guaranteed save — the magic number is a
|
|
438
|
+
// SECOND, organic save (MIN 1 makes the hook a no-op: zero-save cohort is
|
|
439
|
+
// empty). Cohort support comes from save item's event weight (10), which puts
|
|
440
|
+
// the saver cohort at ~100 of ~585 eligible born users at full fidelity —
|
|
441
|
+
// comfortably above the stories' minCohort 50 gate.
|
|
442
|
+
const SAVE_RETENTION_MIN = 2;
|
|
443
|
+
const SAVE_RETENTION_WINDOW_DAYS = 10;
|
|
444
|
+
const SAVE_RETENTION_CUTOFF_DAYS = 25;
|
|
445
|
+
const SAVE_RETENTION_DROP_LIKELIHOOD = 70;
|
|
446
|
+
const QUALITY_FACTORS = {
|
|
447
|
+
"2160p": 1.5,
|
|
448
|
+
"1440p": 1.4,
|
|
449
|
+
"1080p": 1.3,
|
|
450
|
+
"720p": 1.15,
|
|
451
|
+
"480p": 1.0,
|
|
452
|
+
"360p": 0.85,
|
|
453
|
+
"240p": 0.7,
|
|
454
|
+
};
|
|
455
|
+
|
|
456
|
+
// ── DATA ARRAYS ──
|
|
457
|
+
const videoCategories = ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"];
|
|
458
|
+
const spiritAnimals = ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"];
|
|
459
|
+
const productCategories = ["electronics", "books", "clothing", "home", "garden", "toys", "sports", "automotive", "beauty", "health", "grocery", "jewelry", "shoes", "tools", "office supplies"];
|
|
460
|
+
const productDescriptors = ["brand new", "open box", "refurbished", "used", "like new", "vintage", "antique", "collectible"];
|
|
461
|
+
const productSuffixes = ["item", "product", "good", "merchandise", "thing", "object", "widget", "gadget", "device", "apparatus", "contraption", "instrument", "tool", "implement", "utensil", "appliance", "machine", "equipment", "gear", "kit", "set", "package"];
|
|
462
|
+
const assetPreview = [".png", ".jpg", ".jpeg", ".heic", ".mp4", ".mov", ".avi"];
|
|
463
|
+
|
|
464
|
+
// ── HELPER FUNCTIONS ──
|
|
465
|
+
function makeProducts(maxItems = 5) {
|
|
466
|
+
return function () {
|
|
467
|
+
const data = [];
|
|
468
|
+
const numOfItems = integer(1, maxItems);
|
|
469
|
+
|
|
470
|
+
for (let i = 0; i < numOfItems; i++) {
|
|
471
|
+
const category = chance.pickone(productCategories);
|
|
472
|
+
const descriptor = chance.pickone(productDescriptors);
|
|
473
|
+
const suffixWord = chance.pickone(productSuffixes);
|
|
474
|
+
const slug = `${descriptor.replace(/\s+/g, "-").toLowerCase()}-${suffixWord.replace(/\s+/g, "-").toLowerCase()}`;
|
|
475
|
+
const asset = chance.pickone(assetPreview);
|
|
476
|
+
|
|
477
|
+
const price = integer(1, 100);
|
|
478
|
+
const quantity = integer(1, 5);
|
|
479
|
+
|
|
480
|
+
data.push({
|
|
481
|
+
amount: price,
|
|
482
|
+
quantity: quantity,
|
|
483
|
+
total_value: price * quantity,
|
|
484
|
+
featured: chance.pickone([true, false, false]),
|
|
485
|
+
category: category,
|
|
486
|
+
descriptor: descriptor,
|
|
487
|
+
slug: slug,
|
|
488
|
+
assetPreview: `https://example.com/assets/${slug}${asset}`,
|
|
489
|
+
assetType: asset,
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
return () => [data];
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function handleFunnelPreHooks(record, meta) {
|
|
498
|
+
// H9: Dark Theme Power Users — dark +30%, light -15%, custom unchanged
|
|
499
|
+
const isPurchaseFunnel = meta.funnel?.sequence?.includes("checkout");
|
|
500
|
+
if (isPurchaseFunnel) {
|
|
501
|
+
const theme = meta.profile?.theme;
|
|
502
|
+
if (theme === "dark") record.conversionRate = Math.min(95, Math.round(record.conversionRate * 1.3));
|
|
503
|
+
else if (theme === "light") record.conversionRate = Math.round(record.conversionRate * 0.85);
|
|
504
|
+
}
|
|
505
|
+
return record;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function handleEventHooks(record) {
|
|
509
|
+
// H5: Item flattening — promote first item's fields to top level
|
|
510
|
+
if (record.item && Array.isArray(record.item)) {
|
|
511
|
+
record = { ...record, ...record.item[0] };
|
|
512
|
+
delete record.item;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// H3: Toys + Shoes cart correlation
|
|
516
|
+
if (record.event === "checkout" && Array.isArray(record.cart)) {
|
|
517
|
+
const hasToys = record.cart.some(item => item.category === "toys");
|
|
518
|
+
const hasShoes = record.cart.some(item => item.category === "shoes");
|
|
519
|
+
if (hasToys && !hasShoes) {
|
|
520
|
+
const bigCart = makeProducts(20)()()[0];
|
|
521
|
+
const shoeItems = bigCart.filter(item => item.category === "shoes");
|
|
522
|
+
if (shoeItems.length > 0) record.cart.push(shoeItems[0]);
|
|
523
|
+
}
|
|
524
|
+
if (hasShoes && !hasToys) {
|
|
525
|
+
const bigCart = makeProducts(20)()()[0];
|
|
526
|
+
const toyItems = bigCart.filter(item => item.category === "toys");
|
|
527
|
+
if (toyItems.length > 0) record.cart.push(toyItems[0]);
|
|
528
|
+
}
|
|
529
|
+
if (!hasToys && !hasShoes) {
|
|
530
|
+
const cheapFactor = decimal(0.75, 0.9);
|
|
531
|
+
record.cart = record.cart.map(item => ({
|
|
532
|
+
...item,
|
|
533
|
+
amount: Math.round(item.amount * cheapFactor),
|
|
534
|
+
total_value: Math.round(item.total_value * cheapFactor),
|
|
535
|
+
}));
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// H4: Quality → watch time correlation
|
|
540
|
+
if (record.event === "watch video") {
|
|
541
|
+
const quality = record.quality || "480p";
|
|
542
|
+
const factor = QUALITY_FACTORS[quality] ?? 1.0;
|
|
543
|
+
record.watchTimeSec = Math.round(record.watchTimeSec * factor);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
return record;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
function handleEverythingHooks(record, meta) {
|
|
550
|
+
const profile = meta.profile;
|
|
551
|
+
record.forEach(e => {
|
|
552
|
+
e.theme = profile.theme;
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
const datasetEnd = dayjs.unix(meta.datasetEnd);
|
|
556
|
+
const DAY_SIGNUPS_IMPROVED = datasetEnd.subtract(SIGNUP_FIX_DAYS_AGO, "day");
|
|
557
|
+
const DAY_WATCH_TIME_WENT_UP = datasetEnd.subtract(WATCH_INFLECTION_DAYS_AGO, "day");
|
|
558
|
+
|
|
559
|
+
// H7: Signup-flow TTC scaled by loyalty tier (gold/plat fast, bronze slow).
|
|
560
|
+
// MUST run before H1: scaleFunnelTTC anchors on the cluster's earliest step
|
|
561
|
+
// and rescales every later offset — including the sign up event itself — so
|
|
562
|
+
// H1's version stamp has to read the FINAL timestamp. With the old H1-first
|
|
563
|
+
// order, boundary-adjacent sign ups drifted across the fix instant after
|
|
564
|
+
// stamping (3/1719 purity violations at full fidelity). This block consumes
|
|
565
|
+
// no RNG, so the reorder does not perturb the seeded chance stream.
|
|
566
|
+
{
|
|
567
|
+
let loyaltyTier = "silver";
|
|
568
|
+
const scdEntries = meta?.scd?.loyalty_tier;
|
|
569
|
+
if (Array.isArray(scdEntries) && scdEntries.length > 0) {
|
|
570
|
+
const latest = scdEntries.reduce((a, b) => (a.time > b.time ? a : b));
|
|
571
|
+
loyaltyTier = latest.loyalty_tier || "silver";
|
|
572
|
+
} else {
|
|
573
|
+
const uidStr = record[0]?.user_id || "";
|
|
574
|
+
const hash = uidStr.split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
575
|
+
const bucket = hash % 100;
|
|
576
|
+
loyaltyTier = bucket < 20 ? "gold" : bucket < 50 ? "silver" : "bronze";
|
|
577
|
+
}
|
|
578
|
+
const ttcFactor = (
|
|
579
|
+
loyaltyTier === "gold" || loyaltyTier === "platinum" ? LOYALTY_TTC_FAST :
|
|
580
|
+
loyaltyTier === "bronze" ? LOYALTY_TTC_SLOW :
|
|
581
|
+
1.0
|
|
582
|
+
);
|
|
583
|
+
if (ttcFactor !== 1.0) {
|
|
584
|
+
const funnelSteps = new Set(["page view", "view item", "save item", "sign up"]);
|
|
585
|
+
const sorted = record.slice().sort((a, b) => new Date(a.time) - new Date(b.time));
|
|
586
|
+
const signupEvent = sorted.find(e => e.event === "sign up");
|
|
587
|
+
if (signupEvent) {
|
|
588
|
+
const signupMs = new Date(signupEvent.time).getTime();
|
|
589
|
+
const windowMs = LOYALTY_LOOKBACK_DAYS * 24 * 60 * 60 * 1000;
|
|
590
|
+
const funnelEvents = sorted.filter(e =>
|
|
591
|
+
funnelSteps.has(e.event) &&
|
|
592
|
+
new Date(e.time).getTime() >= signupMs - windowMs &&
|
|
593
|
+
new Date(e.time).getTime() <= signupMs
|
|
594
|
+
);
|
|
595
|
+
if (funnelEvents.length >= 2) scaleFunnelTTC(funnelEvents, ttcFactor);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// H1: Signup flow v1 → v2; pre-fix v1 signups lose 50% to _drop
|
|
601
|
+
record.forEach(e => {
|
|
602
|
+
if (e.event !== "sign up") return;
|
|
603
|
+
const eventTime = dayjs(e.time);
|
|
604
|
+
e.signup_flow = "v1";
|
|
605
|
+
if (eventTime.isBefore(DAY_SIGNUPS_IMPROVED)) {
|
|
606
|
+
if (chance.bool({ likelihood: 50 })) e._drop = true;
|
|
607
|
+
}
|
|
608
|
+
if (eventTime.isAfter(DAY_SIGNUPS_IMPROVED)) {
|
|
609
|
+
e.signup_flow = "v2";
|
|
610
|
+
}
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
// H2: Watch time inflection — before: shrink, after: grow
|
|
614
|
+
record.forEach(e => {
|
|
615
|
+
if (e.event !== "watch video") return;
|
|
616
|
+
const eventTime = dayjs(e.time);
|
|
617
|
+
const factor = decimal(WATCH_FACTOR_MIN, WATCH_FACTOR_MAX);
|
|
618
|
+
if (eventTime.isBefore(DAY_WATCH_TIME_WENT_UP)) {
|
|
619
|
+
e.watchTimeSec = Math.round(e.watchTimeSec * (1 - factor));
|
|
620
|
+
}
|
|
621
|
+
if (eventTime.isAfter(DAY_WATCH_TIME_WENT_UP)) {
|
|
622
|
+
e.watchTimeSec = Math.round(e.watchTimeSec * (1 + factor));
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
|
|
626
|
+
// H6: View-item magic number — sweet spot (3-8) boost; over (9+) checkout drop
|
|
627
|
+
const viewItemCount = record.filter(e => e.event === "view item").length;
|
|
628
|
+
if (viewItemCount >= VIEW_SWEET_MIN && viewItemCount <= VIEW_SWEET_MAX) {
|
|
629
|
+
record.forEach(e => {
|
|
630
|
+
if (e.event === "checkout" && Array.isArray(e.cart)) {
|
|
631
|
+
e.cart = e.cart.map(it => ({
|
|
632
|
+
...it,
|
|
633
|
+
amount: typeof it.amount === "number" ? Math.round(it.amount * SWEET_CART_BOOST) : it.amount,
|
|
634
|
+
total_value: typeof it.total_value === "number" ? Math.round(it.total_value * SWEET_CART_BOOST) : it.total_value,
|
|
635
|
+
}));
|
|
636
|
+
}
|
|
637
|
+
});
|
|
638
|
+
const addToCartEvents = record.filter(e => e.event === "add to cart");
|
|
639
|
+
const clones = [];
|
|
640
|
+
addToCartEvents.forEach(e => {
|
|
641
|
+
if (chance.bool({ likelihood: SWEET_CLONE_LIKELIHOOD })) {
|
|
642
|
+
const offsetMs = integer(60_000, 600_000);
|
|
643
|
+
clones.push({
|
|
644
|
+
...e,
|
|
645
|
+
insert_id: uid(),
|
|
646
|
+
time: dayjs(e.time).add(offsetMs, "milliseconds").toISOString(),
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
});
|
|
650
|
+
if (clones.length > 0) record.push(...clones);
|
|
651
|
+
} else if (viewItemCount >= VIEW_OVER_THRESHOLD) {
|
|
652
|
+
for (let i = record.length - 1; i >= 0; i--) {
|
|
653
|
+
if (record[i].event === "checkout" && chance.bool({ likelihood: OVER_CHECKOUT_DROP_LIKELIHOOD })) {
|
|
654
|
+
record.splice(i, 1);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
// H10: Save-item retention — born-in users below threshold churn after day 25
|
|
660
|
+
if (meta.userIsBornInDataset) {
|
|
661
|
+
const firstT = record[0]?.time;
|
|
662
|
+
if (firstT) {
|
|
663
|
+
const window10 = dayjs(firstT).add(SAVE_RETENTION_WINDOW_DAYS, "days").toISOString();
|
|
664
|
+
const saves = record.filter(e => e.event === "save item" && e.time <= window10).length;
|
|
665
|
+
if (saves < SAVE_RETENTION_MIN) {
|
|
666
|
+
const cutoff = dayjs(firstT).add(SAVE_RETENTION_CUTOFF_DAYS, "days");
|
|
667
|
+
for (let i = record.length - 1; i >= 0; i--) {
|
|
668
|
+
if (dayjs(record[i].time).isAfter(cutoff) && chance.bool({ likelihood: SAVE_RETENTION_DROP_LIKELIHOOD })) {
|
|
669
|
+
record.splice(i, 1);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
return record.filter(e => !e._drop);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// ── CONFIG ──
|
|
680
|
+
/** @type {Config} */
|
|
681
|
+
const config = {
|
|
682
|
+
version: 2,
|
|
683
|
+
seed: SEED,
|
|
684
|
+
datasetStart: DATASET_START,
|
|
685
|
+
datasetEnd: DATASET_END,
|
|
686
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
687
|
+
numUsers: NUM_USERS,
|
|
688
|
+
format: "json",
|
|
689
|
+
credentials: {
|
|
690
|
+
token,
|
|
691
|
+
region: "US",
|
|
692
|
+
},
|
|
693
|
+
switches: {
|
|
694
|
+
hasSessionIds: true,
|
|
695
|
+
hasAdSpend: false,
|
|
696
|
+
hasLocation: true,
|
|
697
|
+
hasAndroidDevices: true,
|
|
698
|
+
hasIOSDevices: true,
|
|
699
|
+
hasDesktopDevices: true,
|
|
700
|
+
hasBrowser: true,
|
|
701
|
+
hasCampaigns: false,
|
|
702
|
+
isAnonymous: false,
|
|
703
|
+
alsoInferFunnels: false,
|
|
704
|
+
},
|
|
705
|
+
identity: {
|
|
706
|
+
avgDevicePerUser: 2,
|
|
707
|
+
},
|
|
708
|
+
concurrency: 1,
|
|
709
|
+
events: [
|
|
710
|
+
{
|
|
711
|
+
event: "checkout",
|
|
712
|
+
weight: 2,
|
|
713
|
+
isStrictEvent: false,
|
|
714
|
+
properties: {
|
|
715
|
+
currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
|
|
716
|
+
coupon: weighChoices(["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"]),
|
|
717
|
+
cart: makeProducts(),
|
|
718
|
+
},
|
|
719
|
+
},
|
|
720
|
+
{
|
|
721
|
+
event: "add to cart",
|
|
722
|
+
weight: 4,
|
|
723
|
+
isStrictEvent: false,
|
|
724
|
+
properties: {
|
|
725
|
+
item: makeProducts(1),
|
|
726
|
+
amount: weighNumRange(1, 100, 0.3),
|
|
727
|
+
quantity: weighNumRange(1, 5, 0.3),
|
|
728
|
+
total_value: weighNumRange(1, 500, 0.3),
|
|
729
|
+
featured: [true, false, false],
|
|
730
|
+
category: productCategories,
|
|
731
|
+
descriptor: productDescriptors,
|
|
732
|
+
slug: ["item"],
|
|
733
|
+
assetPreview: ["https://example.com/assets/item.png"],
|
|
734
|
+
assetType: assetPreview,
|
|
735
|
+
},
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
event: "view item",
|
|
739
|
+
weight: 8,
|
|
740
|
+
isStrictEvent: false,
|
|
741
|
+
properties: {
|
|
742
|
+
item: makeProducts(1),
|
|
743
|
+
amount: weighNumRange(1, 100, 0.3),
|
|
744
|
+
quantity: weighNumRange(1, 5, 0.3),
|
|
745
|
+
total_value: weighNumRange(1, 500, 0.3),
|
|
746
|
+
featured: [true, false, false],
|
|
747
|
+
category: productCategories,
|
|
748
|
+
descriptor: productDescriptors,
|
|
749
|
+
slug: ["item"],
|
|
750
|
+
assetPreview: ["https://example.com/assets/item.png"],
|
|
751
|
+
assetType: assetPreview,
|
|
752
|
+
},
|
|
753
|
+
},
|
|
754
|
+
{
|
|
755
|
+
event: "save item",
|
|
756
|
+
// weight 10 (not 5): H10's saver cohort = born users with a 2nd,
|
|
757
|
+
// ORGANIC save in their first 10 days (the Signup Flow funnel
|
|
758
|
+
// guarantees the 1st). At weight 5 the organic save rate left only
|
|
759
|
+
// ~48-67 savers at full fidelity, straddling the stories'
|
|
760
|
+
// minCohort 50 support gate; weight 10 doubles the organic rate.
|
|
761
|
+
weight: 10,
|
|
762
|
+
isStrictEvent: false,
|
|
763
|
+
properties: {
|
|
764
|
+
item: makeProducts(1),
|
|
765
|
+
amount: weighNumRange(1, 100, 0.3),
|
|
766
|
+
quantity: weighNumRange(1, 5, 0.3),
|
|
767
|
+
total_value: weighNumRange(1, 500, 0.3),
|
|
768
|
+
featured: [true, false, false],
|
|
769
|
+
category: productCategories,
|
|
770
|
+
descriptor: productDescriptors,
|
|
771
|
+
slug: ["item"],
|
|
772
|
+
assetPreview: ["https://example.com/assets/item.png"],
|
|
773
|
+
assetType: assetPreview,
|
|
774
|
+
},
|
|
775
|
+
},
|
|
776
|
+
{
|
|
777
|
+
event: "page view",
|
|
778
|
+
weight: 10,
|
|
779
|
+
isStrictEvent: false,
|
|
780
|
+
properties: {
|
|
781
|
+
page: ["/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
|
|
782
|
+
},
|
|
783
|
+
},
|
|
784
|
+
{
|
|
785
|
+
event: "watch video",
|
|
786
|
+
weight: 8,
|
|
787
|
+
isStrictEvent: false,
|
|
788
|
+
properties: {
|
|
789
|
+
watchTimeSec: weighNumRange(10, 600, 0.25),
|
|
790
|
+
quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
|
|
791
|
+
format: ["mp4", "avi", "mov", "mpg"],
|
|
792
|
+
uploader_id: chance.guid.bind(chance),
|
|
793
|
+
},
|
|
794
|
+
},
|
|
795
|
+
{ event: "like video", weight: 6, properties: {} },
|
|
796
|
+
{ event: "dislike video", weight: 4, properties: {} },
|
|
797
|
+
{
|
|
798
|
+
event: "sign up",
|
|
799
|
+
weight: 1,
|
|
800
|
+
isFirstEvent: true,
|
|
801
|
+
isAuthEvent: true,
|
|
802
|
+
properties: {
|
|
803
|
+
signupMethod: ["email", "google", "facebook", "twitter", "linkedin", "github"],
|
|
804
|
+
referral: weighChoices(["none", "none", "none", "friend", "ad", "ad", "ad", "friend", "friend", "friend", "friend"]),
|
|
805
|
+
signup_flow: ["v1"],
|
|
806
|
+
},
|
|
807
|
+
},
|
|
808
|
+
],
|
|
809
|
+
funnels: [
|
|
810
|
+
{
|
|
811
|
+
sequence: ["page view", "view item", "save item", "page view", "sign up"],
|
|
812
|
+
conversionRate: 50,
|
|
813
|
+
order: "first-and-last-fixed",
|
|
814
|
+
weight: 1,
|
|
815
|
+
isFirstFunnel: true,
|
|
816
|
+
timeToConvert: 2,
|
|
817
|
+
experiment: false,
|
|
818
|
+
name: "Signup Flow",
|
|
819
|
+
},
|
|
820
|
+
{
|
|
821
|
+
sequence: ["watch video", "like video", "watch video", "like video"],
|
|
822
|
+
name: "Video Likes",
|
|
823
|
+
conversionRate: 60,
|
|
824
|
+
props: {
|
|
825
|
+
videoCategory: videoCategories,
|
|
826
|
+
quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
|
|
827
|
+
format: ["mp4", "avi", "mov", "mpg"],
|
|
828
|
+
uploader_id: chance.guid.bind(chance),
|
|
829
|
+
},
|
|
830
|
+
},
|
|
831
|
+
{
|
|
832
|
+
name: "Video Dislikes",
|
|
833
|
+
sequence: ["watch video", "dislike video", "watch video", "dislike video"],
|
|
834
|
+
conversionRate: 20,
|
|
835
|
+
props: {
|
|
836
|
+
videoCategory: videoCategories,
|
|
837
|
+
quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
|
|
838
|
+
format: ["mp4", "avi", "mov", "mpg"],
|
|
839
|
+
uploader_id: chance.guid.bind(chance),
|
|
840
|
+
},
|
|
841
|
+
},
|
|
842
|
+
{
|
|
843
|
+
name: "eCommerce Purchase",
|
|
844
|
+
sequence: ["view item", "view item", "add to cart", "view item", "add to cart", "checkout"],
|
|
845
|
+
conversionRate: 15,
|
|
846
|
+
requireRepeats: true,
|
|
847
|
+
weight: 10,
|
|
848
|
+
order: "last-fixed",
|
|
849
|
+
experiment: {
|
|
850
|
+
name: "Express Checkout",
|
|
851
|
+
variants: [
|
|
852
|
+
{ name: "Control" },
|
|
853
|
+
{ name: "Express Checkout", conversionMultiplier: 1.25 },
|
|
854
|
+
{ name: "Social Proof", conversionMultiplier: 1.15, ttcMultiplier: 0.9 },
|
|
855
|
+
],
|
|
856
|
+
startDaysBeforeEnd: 30,
|
|
857
|
+
},
|
|
858
|
+
},
|
|
859
|
+
],
|
|
860
|
+
superProps: {
|
|
861
|
+
theme: ["light", "dark", "custom", "light", "dark"],
|
|
862
|
+
},
|
|
863
|
+
userProps: {
|
|
864
|
+
title: chance.profession.bind(chance),
|
|
865
|
+
luckyNumber: weighNumRange(42, 420, 0.3),
|
|
866
|
+
spiritAnimal: spiritAnimals,
|
|
867
|
+
theme: ["light", "dark", "custom", "light", "dark"],
|
|
868
|
+
},
|
|
869
|
+
scdProps: {
|
|
870
|
+
loyalty_tier: {
|
|
871
|
+
values: ["bronze", "silver", "gold", "platinum"],
|
|
872
|
+
frequency: "month",
|
|
873
|
+
timing: "fuzzy",
|
|
874
|
+
max: 8,
|
|
875
|
+
},
|
|
876
|
+
},
|
|
877
|
+
mirrorProps: {},
|
|
878
|
+
groupKeys: [],
|
|
879
|
+
groupProps: {},
|
|
880
|
+
lookupTables: [],
|
|
881
|
+
hook(record, type, meta) {
|
|
882
|
+
if (type === "funnel-pre") return handleFunnelPreHooks(record, meta);
|
|
883
|
+
if (type === "event") return handleEventHooks(record);
|
|
884
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
885
|
+
return record;
|
|
886
|
+
},
|
|
887
|
+
};
|
|
888
|
+
|
|
889
|
+
export default config;
|
|
890
|
+
|
|
891
|
+
// ── STORIES ──
|
|
892
|
+
// Machine-checkable contract for the 10 hooks above. Thresholds derive from
|
|
893
|
+
// the knob constants (and the engine's experiment/theme composition rules),
|
|
894
|
+
// never from observed output. duckdb assertions run in disk mode only
|
|
895
|
+
// (scripts/verify-stories.mjs after scripts/verify-runner.mjs).
|
|
896
|
+
|
|
897
|
+
const EV = `read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true)`;
|
|
898
|
+
const US = `read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)`;
|
|
899
|
+
const SCDT = `read_json_auto('{{PREFIX}}-loyalty_tier-SCD*.json', sample_size=-1, union_by_name=true)`;
|
|
900
|
+
|
|
901
|
+
// Identity prelude: avgDevicePerUser: 2 + sign up isAuthEvent → born-in
|
|
902
|
+
// users' pre-auth Signup Flow steps carry device_id ONLY. Any per-user
|
|
903
|
+
// aggregation touching signup-funnel steps (H6 bins, H7 TTC, H10 born
|
|
904
|
+
// cohorts) must resolve through the profile's device pool ("anonymousIds"
|
|
905
|
+
// is the legacy USERS-shard key; buildIdentityMap reads the same field).
|
|
906
|
+
const ID_CTE = `dmap AS (SELECT unnest("anonymousIds") AS device_id, distinct_id FROM ${US}),
|
|
907
|
+
ev AS (SELECT coalesce(m.distinct_id::VARCHAR, e.user_id::VARCHAR, e.device_id::VARCHAR) AS uid,
|
|
908
|
+
e.time::TIMESTAMP AS t, e.* FROM ${EV} e LEFT JOIN dmap m ON e.device_id = m.device_id)`;
|
|
909
|
+
|
|
910
|
+
// Temporal boundaries computed from the same knobs the hooks use.
|
|
911
|
+
const FIX_TS = dayjs.utc(DATASET_END).subtract(SIGNUP_FIX_DAYS_AGO, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
912
|
+
const WATCH_TS = dayjs.utc(DATASET_END).subtract(WATCH_INFLECTION_DAYS_AGO, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
913
|
+
const END_TS = dayjs.utc(DATASET_END).format("YYYY-MM-DD HH:mm:ss");
|
|
914
|
+
const WINDOW_DAYS = Math.round(dayjs.utc(DATASET_END).diff(dayjs.utc(DATASET_START), "day", true));
|
|
915
|
+
const PRE_FIX_DAYS = WINDOW_DAYS - SIGNUP_FIX_DAYS_AGO;
|
|
916
|
+
|
|
917
|
+
// Per-user view-item bins shared by the three H6 assertions. LEFT JOIN from
|
|
918
|
+
// profiles so zero-view users land in the low bin; uid comes through the
|
|
919
|
+
// identity prelude so device-only signup-funnel views count.
|
|
920
|
+
const BIN_CTE = `vc AS (SELECT u.distinct_id::VARCHAR AS duid,
|
|
921
|
+
count(e.uid) FILTER (WHERE e.event = 'view item') AS views,
|
|
922
|
+
count(e.uid) FILTER (WHERE e.event = 'add to cart') AS carts,
|
|
923
|
+
count(e.uid) FILTER (WHERE e.event = 'checkout') AS cks
|
|
924
|
+
FROM ${US} u LEFT JOIN ev e ON e.uid = u.distinct_id::VARCHAR GROUP BY 1),
|
|
925
|
+
bins AS (SELECT duid, views, carts, cks,
|
|
926
|
+
CASE WHEN views BETWEEN ${VIEW_SWEET_MIN} AND ${VIEW_SWEET_MAX} THEN 'sweet'
|
|
927
|
+
WHEN views < ${VIEW_SWEET_MIN} THEN 'low' ELSE 'over' END AS bin FROM vc)`;
|
|
928
|
+
|
|
929
|
+
// Strict-paired experiment conversion. Naive time-window pairing (any
|
|
930
|
+
// checkout within 75min of $experiment_started) is POLLUTED here: checkout
|
|
931
|
+
// is also a weight-2 organic event that clusters in the same TimeSoup
|
|
932
|
+
// session as the attempt (~48K organic vs ~2K funnel checkouts at full
|
|
933
|
+
// fidelity), which adds a variant-independent conversion floor and pulls
|
|
934
|
+
// mean TTC from ~60min (full timeToConvert, addTimingOffsets in
|
|
935
|
+
// lib/generators/funnels.js accumulates to ttc on the last step) down to
|
|
936
|
+
// ~34min — compressing both variant ratios toward 1. A converted purchase
|
|
937
|
+
// pass ALWAYS emits its 5 view/add steps between $experiment_started and
|
|
938
|
+
// checkout (applyOrderingStrategy pins checkout last); a failed pass emits
|
|
939
|
+
// integer(1, totalSteps-1) = 1-5 steps (determineConversion), so only ~1/5
|
|
940
|
+
// of failures even have 5 steps to combine with a same-session organic
|
|
941
|
+
// checkout (<1% false-pair rate vs an 11-14% signal). Requiring >= 5
|
|
942
|
+
// intermediate view/add steps therefore isolates true funnel conversions.
|
|
943
|
+
// H6's over-bin checkout drop removes conversions uniformly across variants
|
|
944
|
+
// and cancels in the Express/Control ratio.
|
|
945
|
+
const EXP_CTE = `att AS (SELECT user_id::VARCHAR AS uid, time::TIMESTAMP AS t, "Variant name" AS variant
|
|
946
|
+
FROM ${EV} WHERE event = '$experiment_started'),
|
|
947
|
+
ck AS (SELECT user_id::VARCHAR AS uid, time::TIMESTAMP AS t FROM ${EV} WHERE event = 'checkout'),
|
|
948
|
+
naive AS (SELECT a.uid, a.variant, a.t, min(c.t) AS ct
|
|
949
|
+
FROM att a LEFT JOIN ck c ON c.uid = a.uid AND c.t > a.t AND c.t <= a.t + INTERVAL 75 MINUTE
|
|
950
|
+
GROUP BY 1, 2, 3),
|
|
951
|
+
mids AS (SELECT n.uid, n.t, count(*) AS steps
|
|
952
|
+
FROM naive n JOIN ${EV} s ON s.user_id::VARCHAR = n.uid
|
|
953
|
+
AND s.event IN ('view item', 'add to cart')
|
|
954
|
+
AND s.time::TIMESTAMP > n.t AND s.time::TIMESTAMP < n.ct
|
|
955
|
+
WHERE n.ct IS NOT NULL GROUP BY 1, 2),
|
|
956
|
+
paired AS (SELECT n.uid, n.variant, n.t,
|
|
957
|
+
CASE WHEN m.steps >= 5 THEN n.ct ELSE NULL END AS ct
|
|
958
|
+
FROM naive n LEFT JOIN mids m ON m.uid = n.uid AND m.t = n.t),
|
|
959
|
+
by_variant AS (SELECT variant, count(*) AS attempts, count(ct) AS conversions,
|
|
960
|
+
count(ct)::DOUBLE / count(*) AS rate,
|
|
961
|
+
avg(epoch(ct - t)) FILTER (WHERE ct IS NOT NULL) / 60.0 AS mean_ttc_min,
|
|
962
|
+
count(DISTINCT uid) AS user_count FROM paired GROUP BY 1)`;
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* Five-tier verdict for a ratio measured inside a custom assert: NAILED
|
|
966
|
+
* within ±10% of target, STRONG past floor, WEAK direction-correct, INVERSE
|
|
967
|
+
* wrong side of 1, NONE not computable. Mirrors verdictFor() for op '>=' —
|
|
968
|
+
* needed where the select grammar can't express the comparison (exact-zero
|
|
969
|
+
* purity, strict orderings).
|
|
970
|
+
*/
|
|
971
|
+
function ratioVerdict(ratio, target, floor, detail, smallestCohort, minCohort) {
|
|
972
|
+
if (!Number.isFinite(ratio)) return { pass: false, verdict: "NONE", detail: `ratio not computable — ${detail}` };
|
|
973
|
+
let verdict;
|
|
974
|
+
if (Math.abs(ratio - target) <= 0.1 * target) verdict = "NAILED";
|
|
975
|
+
else if (ratio >= floor) verdict = "STRONG";
|
|
976
|
+
else if (ratio > 1) verdict = "WEAK";
|
|
977
|
+
else if (ratio < 1) verdict = "INVERSE";
|
|
978
|
+
else verdict = "NONE";
|
|
979
|
+
if ((verdict === "NAILED" || verdict === "STRONG") && smallestCohort < minCohort) {
|
|
980
|
+
verdict = "WEAK";
|
|
981
|
+
detail += ` — capped: smallest cohort ${smallestCohort} < minCohort ${minCohort}`;
|
|
982
|
+
}
|
|
983
|
+
return { pass: verdict === "NAILED" || verdict === "STRONG", verdict, detail };
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/** @type {import("../../../types").DungeonStory[]} */
|
|
987
|
+
export const stories = [
|
|
988
|
+
{
|
|
989
|
+
id: "H1-signup-fix",
|
|
990
|
+
hook: "H1",
|
|
991
|
+
archetype: "temporal-inflection",
|
|
992
|
+
narrative: `signup_flow flips v1 → v2 at datasetEnd - ${SIGNUP_FIX_DAYS_AGO}d and 50% of pre-fix signups are dropped. Tag purity is exact (the hook branches on the boundary); the daily-rate jump is a composite of the 2x drop and the late-skewed acquisition ramp (measured 6.9x at iteration scale)`,
|
|
993
|
+
assertions: [
|
|
994
|
+
{
|
|
995
|
+
// deterministic purity: zero v2 before the fix, zero v1 strictly
|
|
996
|
+
// after it. Exact-boundary events legitimately stay v1 (the hook
|
|
997
|
+
// uses isBefore/isAfter), hence < and > not <=/>=.
|
|
998
|
+
breakdown: {
|
|
999
|
+
type: "duckdb",
|
|
1000
|
+
sql: `SELECT 'purity' AS grp,
|
|
1001
|
+
count(*) FILTER (WHERE signup_flow = 'v2' AND time::TIMESTAMP < TIMESTAMP '${FIX_TS}') AS v2_pre,
|
|
1002
|
+
count(*) FILTER (WHERE signup_flow = 'v1' AND time::TIMESTAMP > TIMESTAMP '${FIX_TS}') AS v1_post,
|
|
1003
|
+
count(*) AS signups
|
|
1004
|
+
FROM ${EV} WHERE event = 'sign up'`,
|
|
1005
|
+
},
|
|
1006
|
+
assert: (rows) => {
|
|
1007
|
+
const r = (rows || [])[0];
|
|
1008
|
+
if (!r) return { pass: false, verdict: "NONE", detail: "no signup rows" };
|
|
1009
|
+
const clean = Number(r.v2_pre) === 0 && Number(r.v1_post) === 0;
|
|
1010
|
+
return {
|
|
1011
|
+
pass: clean,
|
|
1012
|
+
verdict: clean ? "NAILED" : "INVERSE",
|
|
1013
|
+
detail: `v2_pre=${r.v2_pre} v1_post=${r.v1_post} of ${r.signups} signups`,
|
|
1014
|
+
};
|
|
1015
|
+
},
|
|
1016
|
+
},
|
|
1017
|
+
{
|
|
1018
|
+
// daily-rate jump: the hook alone guarantees 2x (50% pre-fix drop);
|
|
1019
|
+
// the acquisition ramp multiplies on top. Band floor is the pure
|
|
1020
|
+
// knob effect, ceiling bounds the ramp composite.
|
|
1021
|
+
breakdown: {
|
|
1022
|
+
type: "duckdb",
|
|
1023
|
+
sql: `SELECT 'all' AS grp,
|
|
1024
|
+
(count(*) FILTER (WHERE time::TIMESTAMP >= TIMESTAMP '${FIX_TS}') / ${SIGNUP_FIX_DAYS_AGO}.0)
|
|
1025
|
+
/ nullif(count(*) FILTER (WHERE time::TIMESTAMP < TIMESTAMP '${FIX_TS}') / ${PRE_FIX_DAYS}.0, 0) AS daily_ratio,
|
|
1026
|
+
count(*) AS user_count
|
|
1027
|
+
FROM ${EV} WHERE event = 'sign up'`,
|
|
1028
|
+
},
|
|
1029
|
+
select: { all: { where: { grp: "all" } } },
|
|
1030
|
+
expect: { metric: "all.daily_ratio", op: "between", target: [2.0, 12.0] },
|
|
1031
|
+
minCohort: 300,
|
|
1032
|
+
},
|
|
1033
|
+
],
|
|
1034
|
+
},
|
|
1035
|
+
{
|
|
1036
|
+
id: "H2-watch-inflection",
|
|
1037
|
+
hook: "H2",
|
|
1038
|
+
archetype: "temporal-inflection",
|
|
1039
|
+
narrative: `watchTimeSec scaled by (1-f) before datasetEnd - ${WATCH_INFLECTION_DAYS_AGO}d and (1+f) after, f uniform on [${WATCH_FACTOR_MIN}, ${WATCH_FACTOR_MAX}]. Expected post/pre avg ratio = E[1+f]/E[1-f] = 1.52/0.48 ≈ 3.17 (measured 3.19)`,
|
|
1040
|
+
assertions: [
|
|
1041
|
+
{
|
|
1042
|
+
breakdown: {
|
|
1043
|
+
type: "duckdb",
|
|
1044
|
+
sql: `SELECT 'all' AS grp,
|
|
1045
|
+
avg(watchTimeSec) FILTER (WHERE time::TIMESTAMP > TIMESTAMP '${WATCH_TS}')
|
|
1046
|
+
/ nullif(avg(watchTimeSec) FILTER (WHERE time::TIMESTAMP < TIMESTAMP '${WATCH_TS}'), 0) AS ratio,
|
|
1047
|
+
count(*) AS event_count
|
|
1048
|
+
FROM ${EV} WHERE event = 'watch video'`,
|
|
1049
|
+
},
|
|
1050
|
+
select: { all: { where: { grp: "all" } } },
|
|
1051
|
+
// band = knob prediction 3.17 ± sampling/quality-mix noise
|
|
1052
|
+
expect: { metric: "all.ratio", op: "between", target: [2.6, 3.75] },
|
|
1053
|
+
},
|
|
1054
|
+
],
|
|
1055
|
+
},
|
|
1056
|
+
{
|
|
1057
|
+
id: "H3-toys-shoes-basket",
|
|
1058
|
+
hook: "H3",
|
|
1059
|
+
archetype: "bespoke",
|
|
1060
|
+
narrative: `checkout carts with toys get a shoes item injected when the donor cart contains one (~49% success — makeProducts(20) must roll a shoe) and vice versa; carts with neither get all amounts scaled by uniform [0.75, 0.9] (mean 0.825). Predicted P(shoes|toys) ≈ 0.64 vs P(shoes|no toys) ≈ 0.11 (reciprocal injection drains the complement) → ~6x lift`,
|
|
1061
|
+
assertions: [
|
|
1062
|
+
{
|
|
1063
|
+
// carts keyed by insert_id (one row per checkout event)
|
|
1064
|
+
breakdown: {
|
|
1065
|
+
type: "duckdb",
|
|
1066
|
+
sql: `WITH x AS (SELECT insert_id, unnest(cart) AS item FROM ${EV} WHERE event = 'checkout' AND cart IS NOT NULL),
|
|
1067
|
+
flags AS (SELECT insert_id, bool_or(item.category = 'toys') AS has_toys, bool_or(item.category = 'shoes') AS has_shoes,
|
|
1068
|
+
avg(item.amount) AS avg_amt FROM x GROUP BY 1)
|
|
1069
|
+
SELECT 'all' AS grp,
|
|
1070
|
+
(count(*) FILTER (WHERE has_toys AND has_shoes)::DOUBLE / nullif(count(*) FILTER (WHERE has_toys), 0))
|
|
1071
|
+
/ nullif(count(*) FILTER (WHERE has_shoes AND NOT has_toys)::DOUBLE / nullif(count(*) FILTER (WHERE NOT has_toys), 0), 0) AS lift,
|
|
1072
|
+
count(*) AS cart_count
|
|
1073
|
+
FROM flags`,
|
|
1074
|
+
},
|
|
1075
|
+
select: { all: { where: { grp: "all" } } },
|
|
1076
|
+
expect: { metric: "all.lift", op: "between", target: [3.5, 10.5] },
|
|
1077
|
+
},
|
|
1078
|
+
{
|
|
1079
|
+
breakdown: {
|
|
1080
|
+
type: "duckdb",
|
|
1081
|
+
sql: `WITH x AS (SELECT insert_id, unnest(cart) AS item FROM ${EV} WHERE event = 'checkout' AND cart IS NOT NULL),
|
|
1082
|
+
flags AS (SELECT insert_id, bool_or(item.category = 'toys') AS has_toys, bool_or(item.category = 'shoes') AS has_shoes,
|
|
1083
|
+
avg(item.amount) AS avg_amt FROM x GROUP BY 1)
|
|
1084
|
+
SELECT 'all' AS grp,
|
|
1085
|
+
avg(avg_amt) FILTER (WHERE NOT has_toys AND NOT has_shoes)
|
|
1086
|
+
/ nullif(avg(avg_amt) FILTER (WHERE has_toys OR has_shoes), 0) AS discount,
|
|
1087
|
+
count(*) AS cart_count
|
|
1088
|
+
FROM flags`,
|
|
1089
|
+
},
|
|
1090
|
+
select: { all: { where: { grp: "all" } } },
|
|
1091
|
+
// mean(uniform 0.75..0.9) = 0.825; H6's sweet boost hits both
|
|
1092
|
+
// sides in proportion to bin mix and mostly cancels
|
|
1093
|
+
expect: { metric: "all.discount", op: "between", target: [0.76, 0.89] },
|
|
1094
|
+
},
|
|
1095
|
+
],
|
|
1096
|
+
},
|
|
1097
|
+
{
|
|
1098
|
+
id: "H4-quality-watchtime",
|
|
1099
|
+
hook: "H4",
|
|
1100
|
+
archetype: "cohort-prop-scale",
|
|
1101
|
+
narrative: `watchTimeSec multiplied by quality factor (240p 0.7 … 2160p 1.5). H2's temporal factor is quality-blind and cancels in ratios. Expected 2160p/240p = ${QUALITY_FACTORS["2160p"]}/${QUALITY_FACTORS["240p"]} ≈ 2.14 with strict monotonicity across all 7 tiers`,
|
|
1102
|
+
assertions: [
|
|
1103
|
+
{
|
|
1104
|
+
breakdown: {
|
|
1105
|
+
type: "duckdb",
|
|
1106
|
+
sql: `SELECT quality AS grp, avg(watchTimeSec) AS avg_watch, count(*) AS event_count
|
|
1107
|
+
FROM ${EV} WHERE event = 'watch video' GROUP BY 1`,
|
|
1108
|
+
},
|
|
1109
|
+
select: {
|
|
1110
|
+
uhd: { where: { grp: "2160p" } },
|
|
1111
|
+
sd: { where: { grp: "240p" } },
|
|
1112
|
+
},
|
|
1113
|
+
expect: { metric: "uhd.avg_watch / sd.avg_watch", op: "between", target: [1.85, 2.45] },
|
|
1114
|
+
},
|
|
1115
|
+
{
|
|
1116
|
+
breakdown: {
|
|
1117
|
+
type: "duckdb",
|
|
1118
|
+
sql: `SELECT quality AS grp, avg(watchTimeSec) AS avg_watch, count(*) AS event_count
|
|
1119
|
+
FROM ${EV} WHERE event = 'watch video' GROUP BY 1`,
|
|
1120
|
+
},
|
|
1121
|
+
assert: (rows) => {
|
|
1122
|
+
const order = ["240p", "360p", "480p", "720p", "1080p", "1440p", "2160p"];
|
|
1123
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1124
|
+
const means = order.map((q) => by[q]?.avg_watch);
|
|
1125
|
+
if (means.some((m) => !Number.isFinite(m))) return { pass: false, verdict: "NONE", detail: `missing tiers (${Object.keys(by).join(",")})` };
|
|
1126
|
+
let inversions = 0;
|
|
1127
|
+
for (let i = 1; i < means.length; i++) if (means[i] <= means[i - 1]) inversions++;
|
|
1128
|
+
const detail = order.map((q, i) => `${q}=${means[i].toFixed(0)}`).join(" ");
|
|
1129
|
+
const verdict = inversions === 0 ? "NAILED" : inversions === 1 ? "WEAK" : "INVERSE";
|
|
1130
|
+
return { pass: verdict === "NAILED", verdict, detail: `${detail} (${inversions} inversions)` };
|
|
1131
|
+
},
|
|
1132
|
+
},
|
|
1133
|
+
],
|
|
1134
|
+
},
|
|
1135
|
+
{
|
|
1136
|
+
id: "H5-item-flattening",
|
|
1137
|
+
hook: "H5",
|
|
1138
|
+
archetype: "bespoke",
|
|
1139
|
+
narrative: `view item / add to cart / save item get item[0]'s fields spread top-level and the nested item array deleted; checkout's cart stays nested. Discriminator vs schema defaults: flattened slug is always the "<descriptor>-<suffix>" compound (schema default is the literal "item"), and no "item" column survives in the shards`,
|
|
1140
|
+
assertions: [
|
|
1141
|
+
{
|
|
1142
|
+
// the item column must not exist at all — DESCRIBE the shard schema
|
|
1143
|
+
breakdown: {
|
|
1144
|
+
type: "duckdb",
|
|
1145
|
+
sql: `SELECT 'schema' AS grp,
|
|
1146
|
+
(SELECT count(*) FROM (DESCRIBE SELECT * FROM ${EV}) WHERE column_name = 'item') AS item_cols,
|
|
1147
|
+
(SELECT count(*) FROM (DESCRIBE SELECT * FROM ${EV}) WHERE column_name = 'cart') AS cart_cols`,
|
|
1148
|
+
},
|
|
1149
|
+
assert: (rows) => {
|
|
1150
|
+
const r = (rows || [])[0];
|
|
1151
|
+
if (!r) return { pass: false, verdict: "NONE", detail: "no schema rows" };
|
|
1152
|
+
const ok = Number(r.item_cols) === 0 && Number(r.cart_cols) === 1;
|
|
1153
|
+
return {
|
|
1154
|
+
pass: ok,
|
|
1155
|
+
verdict: ok ? "NAILED" : "INVERSE",
|
|
1156
|
+
detail: `item columns=${r.item_cols} (want 0), cart columns=${r.cart_cols} (want 1)`,
|
|
1157
|
+
};
|
|
1158
|
+
},
|
|
1159
|
+
},
|
|
1160
|
+
{
|
|
1161
|
+
// 100% of item-events carry the flattened compound slug + category;
|
|
1162
|
+
// 100% of checkouts keep the nested cart
|
|
1163
|
+
breakdown: {
|
|
1164
|
+
type: "duckdb",
|
|
1165
|
+
sql: `SELECT event AS grp, count(*) AS n,
|
|
1166
|
+
count(*) FILTER (WHERE slug LIKE '%-%') AS compound_slug,
|
|
1167
|
+
count(*) FILTER (WHERE category IS NOT NULL) AS with_category,
|
|
1168
|
+
count(*) FILTER (WHERE cart IS NOT NULL) AS with_cart
|
|
1169
|
+
FROM ${EV} WHERE event IN ('view item', 'add to cart', 'save item', 'checkout') GROUP BY 1`,
|
|
1170
|
+
},
|
|
1171
|
+
assert: (rows) => {
|
|
1172
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1173
|
+
const itemEvents = ["view item", "add to cart", "save item"];
|
|
1174
|
+
const bad = [];
|
|
1175
|
+
for (const evName of itemEvents) {
|
|
1176
|
+
const r = by[evName];
|
|
1177
|
+
if (!r) { bad.push(`${evName}: missing`); continue; }
|
|
1178
|
+
if (Number(r.compound_slug) !== Number(r.n)) bad.push(`${evName}: slug ${r.compound_slug}/${r.n}`);
|
|
1179
|
+
if (Number(r.with_category) !== Number(r.n)) bad.push(`${evName}: category ${r.with_category}/${r.n}`);
|
|
1180
|
+
}
|
|
1181
|
+
const ck = by["checkout"];
|
|
1182
|
+
if (!ck) bad.push("checkout: missing");
|
|
1183
|
+
else if (Number(ck.with_cart) !== Number(ck.n)) bad.push(`checkout: cart ${ck.with_cart}/${ck.n}`);
|
|
1184
|
+
return {
|
|
1185
|
+
pass: bad.length === 0,
|
|
1186
|
+
verdict: bad.length === 0 ? "NAILED" : "INVERSE",
|
|
1187
|
+
detail: bad.length === 0 ? `100% flattened across ${itemEvents.length} item-events; checkout cart nested` : bad.join("; "),
|
|
1188
|
+
};
|
|
1189
|
+
},
|
|
1190
|
+
},
|
|
1191
|
+
],
|
|
1192
|
+
},
|
|
1193
|
+
{
|
|
1194
|
+
id: "H6-view-magic-number",
|
|
1195
|
+
hook: "H6",
|
|
1196
|
+
archetype: "frequency-sweet-spot",
|
|
1197
|
+
narrative: `3-8 view items → cart amounts x${SWEET_CART_BOOST} + ${SWEET_CLONE_LIKELIHOOD}% add-to-cart clones; 9+ views → ${OVER_CHECKOUT_DROP_LIKELIHOOD}% of checkouts dropped. View count is funnel-dominated (over bin ≈ 89% of users), so signals are per-item and per-ratio: amount ratio is the clean knob, the two rate ratios are documented composites`,
|
|
1198
|
+
assertions: [
|
|
1199
|
+
{
|
|
1200
|
+
// the boost itself: sweet/over avg cart item amount = 1.25 exactly
|
|
1201
|
+
// (H3's neither-discount hits both bins equally and cancels)
|
|
1202
|
+
breakdown: {
|
|
1203
|
+
type: "duckdb",
|
|
1204
|
+
sql: `WITH ${ID_CTE}, ${BIN_CTE},
|
|
1205
|
+
items AS (SELECT e.uid, unnest(e.cart) AS item FROM ev e WHERE e.event = 'checkout' AND e.cart IS NOT NULL)
|
|
1206
|
+
SELECT b.bin AS grp, avg(i.item.amount) AS avg_amt, count(*) AS item_count, count(DISTINCT i.uid) AS user_count
|
|
1207
|
+
FROM items i JOIN bins b ON i.uid = b.duid GROUP BY 1`,
|
|
1208
|
+
},
|
|
1209
|
+
select: {
|
|
1210
|
+
sweet: { where: { grp: "sweet" } },
|
|
1211
|
+
over: { where: { grp: "over" } },
|
|
1212
|
+
},
|
|
1213
|
+
expect: { metric: "sweet.avg_amt / over.avg_amt", op: "between", target: [1.12, 1.38] },
|
|
1214
|
+
minCohort: 200,
|
|
1215
|
+
},
|
|
1216
|
+
{
|
|
1217
|
+
// clone inflation: 1.45x on sweet's add-to-carts, partially offset
|
|
1218
|
+
// by the over bin's funnel-heavier cart mix → ~1.37 measured
|
|
1219
|
+
breakdown: {
|
|
1220
|
+
type: "duckdb",
|
|
1221
|
+
sql: `WITH ${ID_CTE}, ${BIN_CTE}
|
|
1222
|
+
SELECT bin AS grp, count(*) AS user_count,
|
|
1223
|
+
sum(carts)::DOUBLE / nullif(sum(views), 0) AS carts_per_view,
|
|
1224
|
+
sum(cks)::DOUBLE / nullif(sum(carts), 0) AS ck_per_cart
|
|
1225
|
+
FROM bins GROUP BY 1`,
|
|
1226
|
+
},
|
|
1227
|
+
select: {
|
|
1228
|
+
sweet: { where: { grp: "sweet" } },
|
|
1229
|
+
over: { where: { grp: "over" } },
|
|
1230
|
+
},
|
|
1231
|
+
expect: { metric: "sweet.carts_per_view / over.carts_per_view", op: "between", target: [1.15, 1.6] },
|
|
1232
|
+
minCohort: 1000,
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
// checkout suppression composite: the 30% drop plus failed funnel
|
|
1236
|
+
// attempts parking cart-without-checkout prefixes in the over bin
|
|
1237
|
+
breakdown: {
|
|
1238
|
+
type: "duckdb",
|
|
1239
|
+
sql: `WITH ${ID_CTE}, ${BIN_CTE}
|
|
1240
|
+
SELECT bin AS grp, count(*) AS user_count,
|
|
1241
|
+
sum(carts)::DOUBLE / nullif(sum(views), 0) AS carts_per_view,
|
|
1242
|
+
sum(cks)::DOUBLE / nullif(sum(carts), 0) AS ck_per_cart
|
|
1243
|
+
FROM bins GROUP BY 1`,
|
|
1244
|
+
},
|
|
1245
|
+
select: {
|
|
1246
|
+
sweet: { where: { grp: "sweet" } },
|
|
1247
|
+
over: { where: { grp: "over" } },
|
|
1248
|
+
},
|
|
1249
|
+
expect: { metric: "over.ck_per_cart / sweet.ck_per_cart", op: "between", target: [0.35, 0.75] },
|
|
1250
|
+
minCohort: 1000,
|
|
1251
|
+
},
|
|
1252
|
+
],
|
|
1253
|
+
},
|
|
1254
|
+
{
|
|
1255
|
+
id: "H7-loyalty-signup-ttc",
|
|
1256
|
+
hook: "H7",
|
|
1257
|
+
archetype: "funnel-ttc-by-segment",
|
|
1258
|
+
narrative: `Signup Flow TTC scaled by latest SCD loyalty_tier: gold/platinum x${LOYALTY_TTC_FAST}, bronze x${LOYALTY_TTC_SLOW}, silver untouched. Pure-scale ratio bronze/(gold+plat) = ${(LOYALTY_TTC_SLOW / LOYALTY_TTC_FAST).toFixed(2)}; measured 1.85-2.04 across seeds — organic events inside the ${LOYALTY_LOOKBACK_DAYS}d lookback dilute, while re-windowing on scaled times censors tails, so the median hovers near the pure ratio. Pre-auth steps are device-only → identity prelude required`,
|
|
1259
|
+
assertions: [
|
|
1260
|
+
{
|
|
1261
|
+
breakdown: {
|
|
1262
|
+
type: "duckdb",
|
|
1263
|
+
sql: `WITH ${ID_CTE},
|
|
1264
|
+
latest AS (SELECT distinct_id, loyalty_tier FROM (
|
|
1265
|
+
SELECT distinct_id, loyalty_tier, row_number() OVER (PARTITION BY distinct_id ORDER BY time DESC) AS rn FROM ${SCDT}) WHERE rn = 1),
|
|
1266
|
+
su AS (SELECT uid, min(t) AS st FROM ev WHERE event = 'sign up' GROUP BY 1),
|
|
1267
|
+
steps AS (SELECT s.uid, s.st, min(e.t) AS first_step FROM su s JOIN ev e ON e.uid = s.uid
|
|
1268
|
+
WHERE e.event IN ('page view', 'view item', 'save item')
|
|
1269
|
+
AND e.t >= s.st - INTERVAL ${LOYALTY_LOOKBACK_DAYS} DAY AND e.t <= s.st GROUP BY 1, 2)
|
|
1270
|
+
SELECT CASE WHEN l.loyalty_tier IN ('gold', 'platinum') THEN 'fast'
|
|
1271
|
+
WHEN l.loyalty_tier = 'bronze' THEN 'slow' ELSE 'silver' END AS grp,
|
|
1272
|
+
count(*) AS user_count, median(epoch(st - first_step)) / 60.0 AS med_ttc_min
|
|
1273
|
+
FROM steps s JOIN latest l ON l.distinct_id = s.uid GROUP BY 1`,
|
|
1274
|
+
},
|
|
1275
|
+
select: {
|
|
1276
|
+
slow: { where: { grp: "slow" } },
|
|
1277
|
+
fast: { where: { grp: "fast" } },
|
|
1278
|
+
},
|
|
1279
|
+
expect: { metric: "slow.med_ttc_min / fast.med_ttc_min", op: "between", target: [1.5, 2.3] },
|
|
1280
|
+
minCohort: 200,
|
|
1281
|
+
},
|
|
1282
|
+
{
|
|
1283
|
+
// strict ordering: fast < silver < slow median TTC
|
|
1284
|
+
breakdown: {
|
|
1285
|
+
type: "duckdb",
|
|
1286
|
+
sql: `WITH ${ID_CTE},
|
|
1287
|
+
latest AS (SELECT distinct_id, loyalty_tier FROM (
|
|
1288
|
+
SELECT distinct_id, loyalty_tier, row_number() OVER (PARTITION BY distinct_id ORDER BY time DESC) AS rn FROM ${SCDT}) WHERE rn = 1),
|
|
1289
|
+
su AS (SELECT uid, min(t) AS st FROM ev WHERE event = 'sign up' GROUP BY 1),
|
|
1290
|
+
steps AS (SELECT s.uid, s.st, min(e.t) AS first_step FROM su s JOIN ev e ON e.uid = s.uid
|
|
1291
|
+
WHERE e.event IN ('page view', 'view item', 'save item')
|
|
1292
|
+
AND e.t >= s.st - INTERVAL ${LOYALTY_LOOKBACK_DAYS} DAY AND e.t <= s.st GROUP BY 1, 2)
|
|
1293
|
+
SELECT CASE WHEN l.loyalty_tier IN ('gold', 'platinum') THEN 'fast'
|
|
1294
|
+
WHEN l.loyalty_tier = 'bronze' THEN 'slow' ELSE 'silver' END AS grp,
|
|
1295
|
+
count(*) AS user_count, median(epoch(st - first_step)) / 60.0 AS med_ttc_min
|
|
1296
|
+
FROM steps s JOIN latest l ON l.distinct_id = s.uid GROUP BY 1`,
|
|
1297
|
+
},
|
|
1298
|
+
assert: (rows) => {
|
|
1299
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1300
|
+
const f = by.fast, s = by.silver, sl = by.slow;
|
|
1301
|
+
if (!f || !s || !sl) return { pass: false, verdict: "NONE", detail: `missing tier groups (${Object.keys(by).join(",")})` };
|
|
1302
|
+
const ordered = f.med_ttc_min < s.med_ttc_min && s.med_ttc_min < sl.med_ttc_min;
|
|
1303
|
+
const smallest = Math.min(f.user_count, s.user_count, sl.user_count);
|
|
1304
|
+
let detail = `fast=${f.med_ttc_min.toFixed(1)}m silver=${s.med_ttc_min.toFixed(1)}m slow=${sl.med_ttc_min.toFixed(1)}m`;
|
|
1305
|
+
let verdict = ordered ? "NAILED" : (f.med_ttc_min < sl.med_ttc_min ? "WEAK" : "INVERSE");
|
|
1306
|
+
if (verdict === "NAILED" && smallest < 100) {
|
|
1307
|
+
verdict = "WEAK";
|
|
1308
|
+
detail += ` — capped: smallest cohort ${smallest} < minCohort 100`;
|
|
1309
|
+
}
|
|
1310
|
+
return { pass: verdict === "NAILED", verdict, detail };
|
|
1311
|
+
},
|
|
1312
|
+
},
|
|
1313
|
+
],
|
|
1314
|
+
},
|
|
1315
|
+
{
|
|
1316
|
+
id: "H8-checkout-experiment",
|
|
1317
|
+
hook: "H8",
|
|
1318
|
+
archetype: "experiment-lift",
|
|
1319
|
+
narrative: `engine experiment on eCommerce Purchase: variants assigned by deterministic per-user hash (equal thirds); effective conversionRate = round(base x multiplier), theme-composed (H9 applies after) → Control ≈ 16.2%, Express ≈ 20.2%, Social ≈ 17.8% per attempt; Social also gets ttcMultiplier 0.9 on the funnel's 1h window. "Variant name" lives only on $experiment_started. Conversions measured by STRICT pairing (checkout within 75min AND >= 5 view/add steps in between) — naive time-window pairing is diluted toward ratio 1 by same-session organic checkouts (engine correctness cross-checked by an isolated no-hook repro: Express lift present, Social/Control TTC 0.888 ≈ knob 0.9)`,
|
|
1320
|
+
assertions: [
|
|
1321
|
+
{
|
|
1322
|
+
// hash thirds: each variant's share of experiment users in [0.28, 0.39]
|
|
1323
|
+
breakdown: {
|
|
1324
|
+
type: "duckdb",
|
|
1325
|
+
sql: `WITH ${EXP_CTE} SELECT variant AS grp, attempts, conversions, rate, mean_ttc_min, user_count FROM by_variant`,
|
|
1326
|
+
},
|
|
1327
|
+
assert: (rows) => {
|
|
1328
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1329
|
+
const names = ["Control", "Express Checkout", "Social Proof"];
|
|
1330
|
+
if (names.some((n) => !by[n])) return { pass: false, verdict: "NONE", detail: `missing variants (${Object.keys(by).join(",")})` };
|
|
1331
|
+
const total = names.reduce((acc, n) => acc + Number(by[n].user_count), 0);
|
|
1332
|
+
const shares = names.map((n) => Number(by[n].user_count) / total);
|
|
1333
|
+
const ok = shares.every((s) => s >= 0.28 && s <= 0.39);
|
|
1334
|
+
const smallest = Math.min(...names.map((n) => Number(by[n].user_count)));
|
|
1335
|
+
let detail = names.map((n, i) => `${n}=${(shares[i] * 100).toFixed(1)}%`).join(" ");
|
|
1336
|
+
let verdict = ok ? "NAILED" : shares.every((s) => s >= 0.2 && s <= 0.5) ? "WEAK" : "INVERSE";
|
|
1337
|
+
if (verdict === "NAILED" && smallest < 200) {
|
|
1338
|
+
verdict = "WEAK";
|
|
1339
|
+
detail += ` — capped: smallest cohort ${smallest} < minCohort 200`;
|
|
1340
|
+
}
|
|
1341
|
+
return { pass: verdict === "NAILED", verdict, detail };
|
|
1342
|
+
},
|
|
1343
|
+
},
|
|
1344
|
+
{
|
|
1345
|
+
// Express/Control strict-paired conversion. Point 1.25
|
|
1346
|
+
// (20.2/16.2 theme-composed; H6's uniform checkout drop cancels
|
|
1347
|
+
// in the ratio); band [1.05, 1.55] covers ~1.4 sigma of counting
|
|
1348
|
+
// noise at ~100 strict conversions per variant.
|
|
1349
|
+
breakdown: {
|
|
1350
|
+
type: "duckdb",
|
|
1351
|
+
sql: `WITH ${EXP_CTE} SELECT variant AS grp, attempts, conversions, rate, mean_ttc_min, user_count FROM by_variant`,
|
|
1352
|
+
},
|
|
1353
|
+
select: {
|
|
1354
|
+
express: { where: { grp: "Express Checkout" } },
|
|
1355
|
+
control: { where: { grp: "Control" } },
|
|
1356
|
+
},
|
|
1357
|
+
expect: { metric: "express.rate / control.rate", op: "between", target: [1.05, 1.55] },
|
|
1358
|
+
minCohort: 200,
|
|
1359
|
+
},
|
|
1360
|
+
{
|
|
1361
|
+
// Social Proof ttcMultiplier 0.9 on strict-paired mean TTC —
|
|
1362
|
+
// true conversions land at ~ttc (Control ~60min, Social ~54min),
|
|
1363
|
+
// so the ratio reads the knob directly; band [0.84, 0.96].
|
|
1364
|
+
breakdown: {
|
|
1365
|
+
type: "duckdb",
|
|
1366
|
+
sql: `WITH ${EXP_CTE} SELECT variant AS grp, attempts, conversions, rate, mean_ttc_min, user_count FROM by_variant`,
|
|
1367
|
+
},
|
|
1368
|
+
select: {
|
|
1369
|
+
social: { where: { grp: "Social Proof" } },
|
|
1370
|
+
control: { where: { grp: "Control" } },
|
|
1371
|
+
},
|
|
1372
|
+
expect: { metric: "social.mean_ttc_min / control.mean_ttc_min", op: "between", target: [0.84, 0.96] },
|
|
1373
|
+
minCohort: 200,
|
|
1374
|
+
},
|
|
1375
|
+
],
|
|
1376
|
+
},
|
|
1377
|
+
{
|
|
1378
|
+
id: "H9-dark-theme-power",
|
|
1379
|
+
hook: "H9",
|
|
1380
|
+
archetype: "funnel-conversion-by-segment",
|
|
1381
|
+
narrative: `funnel-pre multiplies purchase-funnel conversionRate: dark round(15x1.3)=20, light round(15x0.85)=13, custom 15 → per-attempt dark/light ≈ 1.54, diluted to ≈ 1.48 checkouts-per-user by theme-blind organic checkouts (weight 2). Theme is stamped on every event by the everything hook`,
|
|
1382
|
+
assertions: [
|
|
1383
|
+
{
|
|
1384
|
+
breakdown: {
|
|
1385
|
+
type: "duckdb",
|
|
1386
|
+
sql: `WITH uc AS (SELECT theme, count(*) AS users FROM ${US} GROUP BY 1),
|
|
1387
|
+
ec AS (SELECT theme, count(*) AS cks FROM ${EV} WHERE event = 'checkout' GROUP BY 1)
|
|
1388
|
+
SELECT uc.theme AS grp, uc.users AS user_count, ec.cks AS checkouts, ec.cks::DOUBLE / uc.users AS per_user
|
|
1389
|
+
FROM uc JOIN ec ON uc.theme = ec.theme`,
|
|
1390
|
+
},
|
|
1391
|
+
select: {
|
|
1392
|
+
dark: { where: { grp: "dark" } },
|
|
1393
|
+
light: { where: { grp: "light" } },
|
|
1394
|
+
},
|
|
1395
|
+
expect: { metric: "dark.per_user / light.per_user", op: "between", target: [1.25, 1.7] },
|
|
1396
|
+
minCohort: 2000,
|
|
1397
|
+
},
|
|
1398
|
+
{
|
|
1399
|
+
// strict ordering dark > custom > light (custom/light knob 15/13 ≈ 1.15)
|
|
1400
|
+
breakdown: {
|
|
1401
|
+
type: "duckdb",
|
|
1402
|
+
sql: `WITH uc AS (SELECT theme, count(*) AS users FROM ${US} GROUP BY 1),
|
|
1403
|
+
ec AS (SELECT theme, count(*) AS cks FROM ${EV} WHERE event = 'checkout' GROUP BY 1)
|
|
1404
|
+
SELECT uc.theme AS grp, uc.users AS user_count, ec.cks::DOUBLE / uc.users AS per_user
|
|
1405
|
+
FROM uc JOIN ec ON uc.theme = ec.theme`,
|
|
1406
|
+
},
|
|
1407
|
+
assert: (rows) => {
|
|
1408
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1409
|
+
const d = by.dark, c = by.custom, l = by.light;
|
|
1410
|
+
if (!d || !c || !l) return { pass: false, verdict: "NONE", detail: `missing themes (${Object.keys(by).join(",")})` };
|
|
1411
|
+
const ordered = d.per_user > c.per_user && c.per_user > l.per_user;
|
|
1412
|
+
const smallest = Math.min(d.user_count, c.user_count, l.user_count);
|
|
1413
|
+
let detail = `dark=${d.per_user.toFixed(3)} custom=${c.per_user.toFixed(3)} light=${l.per_user.toFixed(3)}`;
|
|
1414
|
+
let verdict = ordered ? "NAILED" : (d.per_user > l.per_user ? "WEAK" : "INVERSE");
|
|
1415
|
+
if (verdict === "NAILED" && smallest < 2000) {
|
|
1416
|
+
verdict = "WEAK";
|
|
1417
|
+
detail += ` — capped: smallest cohort ${smallest} < minCohort 2000`;
|
|
1418
|
+
}
|
|
1419
|
+
return { pass: verdict === "NAILED", verdict, detail };
|
|
1420
|
+
},
|
|
1421
|
+
},
|
|
1422
|
+
],
|
|
1423
|
+
},
|
|
1424
|
+
{
|
|
1425
|
+
id: "H10-save-retention",
|
|
1426
|
+
hook: "H10",
|
|
1427
|
+
archetype: "retention-divergence",
|
|
1428
|
+
narrative: `born-in users with fewer than ${SAVE_RETENTION_MIN} save items in their first ${SAVE_RETENTION_WINDOW_DAYS} days lose ${SAVE_RETENTION_DROP_LIKELIHOOD}% of events after day ${SAVE_RETENTION_CUTOFF_DAYS}. The ratio-of-ratios (nonsaver post/pre) / (saver post/pre) cancels window lengths and the acquisition ramp → ≈ ${(1 - SAVE_RETENTION_DROP_LIKELIHOOD / 100).toFixed(2)} by knob. Every signer has one funnel-guaranteed save (Signup Flow includes the step) — savers made a 2nd, organic save (~15-20% of eligible born users)`,
|
|
1429
|
+
assertions: [
|
|
1430
|
+
{
|
|
1431
|
+
breakdown: {
|
|
1432
|
+
type: "duckdb",
|
|
1433
|
+
sql: `WITH ${ID_CTE},
|
|
1434
|
+
born AS (SELECT uid, min(t) AS t0 FROM ev WHERE event = 'sign up' GROUP BY 1),
|
|
1435
|
+
eligible AS (SELECT uid, t0 FROM born
|
|
1436
|
+
WHERE t0 <= TIMESTAMP '${END_TS}' - INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS + SAVE_RETENTION_WINDOW_DAYS} DAY),
|
|
1437
|
+
per AS (SELECT b.uid,
|
|
1438
|
+
count(e.uid) FILTER (WHERE e.event = 'save item' AND e.t <= b.t0 + INTERVAL ${SAVE_RETENTION_WINDOW_DAYS} DAY) AS early_saves,
|
|
1439
|
+
count(e.uid) FILTER (WHERE e.t <= b.t0 + INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS} DAY) AS pre_ev,
|
|
1440
|
+
count(e.uid) FILTER (WHERE e.t > b.t0 + INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS} DAY) AS post_ev
|
|
1441
|
+
FROM eligible b JOIN ev e ON e.uid = b.uid GROUP BY 1)
|
|
1442
|
+
SELECT CASE WHEN early_saves >= ${SAVE_RETENTION_MIN} THEN 'saver' ELSE 'nonsaver' END AS grp,
|
|
1443
|
+
count(*) AS user_count, avg(post_ev)::DOUBLE / nullif(avg(pre_ev), 0) AS post_pre
|
|
1444
|
+
FROM per GROUP BY 1`,
|
|
1445
|
+
},
|
|
1446
|
+
select: {
|
|
1447
|
+
nonsaver: { where: { grp: "nonsaver" } },
|
|
1448
|
+
saver: { where: { grp: "saver" } },
|
|
1449
|
+
},
|
|
1450
|
+
// knob 0.30; band absorbs saver-cohort sampling noise (~85
|
|
1451
|
+
// savers at full fidelity: ~1.8K born signups x ~31% with 35d
|
|
1452
|
+
// runway x ~14% making a 2nd organic save in the 10d window at
|
|
1453
|
+
// save-item weight 10 — measured 21/148 at 12K iteration scale)
|
|
1454
|
+
expect: { metric: "nonsaver.post_pre / saver.post_pre", op: "between", target: [0.15, 0.45] },
|
|
1455
|
+
minCohort: 50,
|
|
1456
|
+
},
|
|
1457
|
+
{
|
|
1458
|
+
// control: savers are untouched — their post/pre rate stays ~flat
|
|
1459
|
+
breakdown: {
|
|
1460
|
+
type: "duckdb",
|
|
1461
|
+
sql: `WITH ${ID_CTE},
|
|
1462
|
+
born AS (SELECT uid, min(t) AS t0 FROM ev WHERE event = 'sign up' GROUP BY 1),
|
|
1463
|
+
eligible AS (SELECT uid, t0 FROM born
|
|
1464
|
+
WHERE t0 <= TIMESTAMP '${END_TS}' - INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS + SAVE_RETENTION_WINDOW_DAYS} DAY),
|
|
1465
|
+
per AS (SELECT b.uid,
|
|
1466
|
+
count(e.uid) FILTER (WHERE e.event = 'save item' AND e.t <= b.t0 + INTERVAL ${SAVE_RETENTION_WINDOW_DAYS} DAY) AS early_saves,
|
|
1467
|
+
count(e.uid) FILTER (WHERE e.t <= b.t0 + INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS} DAY) AS pre_ev,
|
|
1468
|
+
count(e.uid) FILTER (WHERE e.t > b.t0 + INTERVAL ${SAVE_RETENTION_CUTOFF_DAYS} DAY) AS post_ev
|
|
1469
|
+
FROM eligible b JOIN ev e ON e.uid = b.uid GROUP BY 1)
|
|
1470
|
+
SELECT CASE WHEN early_saves >= ${SAVE_RETENTION_MIN} THEN 'saver' ELSE 'nonsaver' END AS grp,
|
|
1471
|
+
count(*) AS user_count, avg(post_ev)::DOUBLE / nullif(avg(pre_ev), 0) AS post_pre
|
|
1472
|
+
FROM per GROUP BY 1`,
|
|
1473
|
+
},
|
|
1474
|
+
select: { saver: { where: { grp: "saver" } } },
|
|
1475
|
+
expect: { metric: "saver.post_pre", op: "between", target: [0.8, 1.35] },
|
|
1476
|
+
minCohort: 50,
|
|
1477
|
+
},
|
|
1478
|
+
],
|
|
1479
|
+
},
|
|
1480
|
+
];
|