@ak--47/dungeon-master 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +96 -0
- package/.claude/skills/create-project/provision.mjs +336 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +211 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +3 -2
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -0,0 +1,1852 @@
|
|
|
1
|
+
// ── IMPORTS ──
|
|
2
|
+
import dayjs from "dayjs";
|
|
3
|
+
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
5
|
+
import "dotenv/config";
|
|
6
|
+
import * as u from "@ak--47/dungeon-master/utils";
|
|
7
|
+
/** @typedef {import("../../../types").Dungeon} Config */
|
|
8
|
+
|
|
9
|
+
// ── OVERVIEW ──
|
|
10
|
+
/*
|
|
11
|
+
* NAME: SafeHaven Insurance
|
|
12
|
+
* APP: Modern insurance web application. Users browse coverage options,
|
|
13
|
+
* request quotes, complete multi-step applications, manage policies,
|
|
14
|
+
* file claims, and make premium payments. Spans auto / home / life /
|
|
15
|
+
* health / renters insurance types across web, iOS, and Android.
|
|
16
|
+
* SCALE: 15,000 users, ~1.8M events, 121 days (2026-01-01 → 2026-05-01)
|
|
17
|
+
* CORE LOOP: account created → quote requested → application submitted → policy activated → payment made
|
|
18
|
+
*
|
|
19
|
+
* EVENTS (18):
|
|
20
|
+
* page viewed (10) > application step completed (6) > quote requested (5)
|
|
21
|
+
* > payment made (5) > quote received (4) > application started (4)
|
|
22
|
+
* > claim status checked (4) > support ticket created (4) > coverage reviewed (4)
|
|
23
|
+
* > document uploaded (3) > application submitted (3) > support ticket resolved (3)
|
|
24
|
+
* > application approved (2) > policy activated (2) > claim filed (2)
|
|
25
|
+
* > profile updated (2) > renewal completed (2) > account created (1)
|
|
26
|
+
*
|
|
27
|
+
* FUNNELS (5):
|
|
28
|
+
* - Onboarding: account created → page viewed → quote requested (85%)
|
|
29
|
+
* - Application Completion: application started → step completed → document uploaded → application submitted (60%)
|
|
30
|
+
* - Application Approval: application submitted → application approved → policy activated (70%)
|
|
31
|
+
* - Claims Process: claim filed → claim status checked → support ticket created (50%, A/B experiment)
|
|
32
|
+
* - Policy Renewal: coverage reviewed → payment made → renewal completed (65%)
|
|
33
|
+
*
|
|
34
|
+
* USER PROPS: Platform, insurance_type, app_version, age_range, risk_profile, policy_count, lifetime_premium, preferred_contact
|
|
35
|
+
* SUPER PROPS: Platform, insurance_type, app_version
|
|
36
|
+
* SCD PROPS: policy_status (pending/active/lapsed/renewed, monthly fuzzy, max 8)
|
|
37
|
+
* GROUPS: none
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
// ── HOOK STORIES ──
|
|
41
|
+
/*
|
|
42
|
+
* NOTE: All cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
43
|
+
* via raw-prop breakdowns (date, app_version, issue_category) or
|
|
44
|
+
* behavioral cohorts.
|
|
45
|
+
*
|
|
46
|
+
* -------------------------------------------------------------------
|
|
47
|
+
* 1. VERSION STAMPING (everything)
|
|
48
|
+
* -------------------------------------------------------------------
|
|
49
|
+
*
|
|
50
|
+
* PATTERN: Every event gets a deterministic app_version based on its
|
|
51
|
+
* final timestamp. All users shift simultaneously on release dates:
|
|
52
|
+
* - Days 0-30: v2.10
|
|
53
|
+
* - Days 30-60: v2.11
|
|
54
|
+
* - Days 60-90: v2.12
|
|
55
|
+
* - Last 10 days: v2.13
|
|
56
|
+
* app_version is a real product property already in superProps.
|
|
57
|
+
*
|
|
58
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
59
|
+
*
|
|
60
|
+
* Report 1: Event Volume by App Version
|
|
61
|
+
* - Report type: Insights
|
|
62
|
+
* - Event: All Events (or "application submitted" — high volume)
|
|
63
|
+
* - Measure: Total
|
|
64
|
+
* - Breakdown: "app_version"
|
|
65
|
+
* - Line chart by day
|
|
66
|
+
* - Expected: clean cutovers between versions, no overlap
|
|
67
|
+
* - NOTE: "page viewed" only fires inside the Onboarding funnel
|
|
68
|
+
* (funnel-member events are excluded from the random-event pool),
|
|
69
|
+
* so it is rare — use a high-volume event for this chart.
|
|
70
|
+
*
|
|
71
|
+
* REAL-WORLD ANALOGUE: Forced auto-update SaaS apps cut all users over
|
|
72
|
+
* on release day, producing crisp version bands.
|
|
73
|
+
*
|
|
74
|
+
* -------------------------------------------------------------------
|
|
75
|
+
* 2. SUPPORT TICKET VOLUME DROP (everything)
|
|
76
|
+
* -------------------------------------------------------------------
|
|
77
|
+
*
|
|
78
|
+
* PATTERN: Pre-v2.13: each user gets 2-3 extra cloned support-ticket
|
|
79
|
+
* created events stamped with bug-related issue_category values
|
|
80
|
+
* (form_crash, login_error, page_timeout, payment_failure — part of the
|
|
81
|
+
* declared issue_category value list). Post-v2.13: tickets progressively
|
|
82
|
+
* removed (30% day 1 → 85% day 10, capped) AND surviving post-v2.13
|
|
83
|
+
* tickets that carry a bug category are reassigned to a non-bug
|
|
84
|
+
* category — the release fixed the bugs, so bug-category share drops to
|
|
85
|
+
* EXACTLY ZERO after the release. No flag — discover via issue_category
|
|
86
|
+
* breakdown over time.
|
|
87
|
+
*
|
|
88
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
89
|
+
*
|
|
90
|
+
* Report 1: Ticket Volume Over Time
|
|
91
|
+
* - Report type: Insights
|
|
92
|
+
* - Event: "support ticket created"
|
|
93
|
+
* - Measure: Total
|
|
94
|
+
* - Breakdown: "app_version"
|
|
95
|
+
* - Line chart by day
|
|
96
|
+
* - Expected: high volume on v2.12, sharp drop on v2.13 to ~0.3x the
|
|
97
|
+
* final-v2.12 rate (injection inflates pre-release ~1.7x organic;
|
|
98
|
+
* ramped removal keeps ~43% post-release)
|
|
99
|
+
*
|
|
100
|
+
* Report 2: Bug Category Share Pre vs Post v2.13
|
|
101
|
+
* - Report type: Insights
|
|
102
|
+
* - Event: "support ticket created"
|
|
103
|
+
* - Measure: Total
|
|
104
|
+
* - Breakdown: "issue_category"
|
|
105
|
+
* - Compare pre-v2.13 vs v2.13 date ranges
|
|
106
|
+
* - Expected: form_crash / login_error / page_timeout / payment_failure
|
|
107
|
+
* hold ~67% share pre-v2.13 (organic 4/9 uniform share plus the
|
|
108
|
+
* all-bug injections) and are EXACTLY ZERO post-v2.13
|
|
109
|
+
*
|
|
110
|
+
* REAL-WORLD ANALOGUE: A UX-fix release reduces ticket volume overnight.
|
|
111
|
+
*
|
|
112
|
+
* -------------------------------------------------------------------
|
|
113
|
+
* 3. ACTIVATION GAP FIXED BY v2.13 (everything)
|
|
114
|
+
* -------------------------------------------------------------------
|
|
115
|
+
*
|
|
116
|
+
* PATTERN: Pre-v2.13: ~95% of users have ALL their pre-release
|
|
117
|
+
* "policy activated" events dropped (per-user gating). Post-v2.13: kept.
|
|
118
|
+
* Deliberately does NOT touch "application approved": Hook #5 pins every
|
|
119
|
+
* approved event to firstStart + 36-63h, which lands almost all of them
|
|
120
|
+
* early in the dataset regardless of when the funnel instance ran —
|
|
121
|
+
* dropping relocated approvals pre-v2.13 would leave no post-v2.13
|
|
122
|
+
* approvals to step up and starve Hook #5's TTC cohort. The era signal
|
|
123
|
+
* lives on "policy activated" alone: approvals went through the whole
|
|
124
|
+
* time, but the activation step was broken until v2.13 shipped.
|
|
125
|
+
* No flag — discover via volume line chart or per-era conversion.
|
|
126
|
+
*
|
|
127
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
128
|
+
*
|
|
129
|
+
* Report 1: Activation Volume Over Time
|
|
130
|
+
* - Report type: Insights
|
|
131
|
+
* - Event: "policy activated"
|
|
132
|
+
* - Measure: Total, line chart by day
|
|
133
|
+
* - Expected: near-flatline pre-v2.13 (~5% survives), then a massive
|
|
134
|
+
* step-up on release day — ~28x the pre-release daily rate
|
|
135
|
+
* (1/0.05 = 20x from the gate, amplified by organic late-dataset
|
|
136
|
+
* activity drift)
|
|
137
|
+
*
|
|
138
|
+
* Report 2: Submit → Activate Conversion by Era
|
|
139
|
+
* - Report type: Funnels
|
|
140
|
+
* - Steps: "application submitted" -> "policy activated"
|
|
141
|
+
* - Breakdown: "app_version" (or compare date ranges)
|
|
142
|
+
* - Expected: pre-v2.13 users convert ~4%, post-v2.13 ~50% — a
|
|
143
|
+
* ~12x step-up (below the 20x volume gate because the 111-day
|
|
144
|
+
* pre-era gives survivors far more chances to show an activation
|
|
145
|
+
* than the 10-day post-era window)
|
|
146
|
+
*
|
|
147
|
+
* REAL-WORLD ANALOGUE: Policy issuance succeeded but the activation
|
|
148
|
+
* step silently failed for nearly everyone until the v2.13 fix.
|
|
149
|
+
*
|
|
150
|
+
* -------------------------------------------------------------------
|
|
151
|
+
* 4. APPLICATION-STEP MAGIC NUMBER (everything)
|
|
152
|
+
* -------------------------------------------------------------------
|
|
153
|
+
*
|
|
154
|
+
* PATTERN: Users with 8-14 application step completed events sit in the
|
|
155
|
+
* sweet spot — approved_premium boosted +35%. Users with 15+ steps
|
|
156
|
+
* are over-engaged (likely fraud or signal review); 40% of their
|
|
157
|
+
* application approved events drop. No flag.
|
|
158
|
+
*
|
|
159
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
160
|
+
*
|
|
161
|
+
* Report 1: Avg Approved Premium by Step Bucket
|
|
162
|
+
* - Report type: Insights (with cohort)
|
|
163
|
+
* - Cohort A: users with 8-14 "application step completed"
|
|
164
|
+
* - Cohort B: users with 0-7
|
|
165
|
+
* - Event: "application approved"
|
|
166
|
+
* - Measure: Average of "approved_premium"
|
|
167
|
+
* - Expected: A ~ 1.35x B
|
|
168
|
+
*
|
|
169
|
+
* Report 2: Approvals per Submitted Application on Heavy Step-Completers
|
|
170
|
+
* - Report type: Insights (with cohort, formula)
|
|
171
|
+
* - Cohort C: users with >= 15 "application step completed"
|
|
172
|
+
* - Cohort A: users with 8-14
|
|
173
|
+
* - Formula: total "application approved" / total "application submitted"
|
|
174
|
+
* - Expected: C ~ 0.5x A. The knob is a 40% approval drop (0.6x), but
|
|
175
|
+
* raw approvals-per-USER hides it: 15+ steppers are hyperactive and
|
|
176
|
+
* organically earn ~2x the approvals. Normalizing by submitted
|
|
177
|
+
* applications exposes the drop; the observed ratio sits slightly
|
|
178
|
+
* below 0.6 because heavy steppers' submit mix skews toward
|
|
179
|
+
* Application Completion conversions, diluting their denominator.
|
|
180
|
+
*
|
|
181
|
+
* REAL-WORLD ANALOGUE: Engaged applicants get higher premiums approved;
|
|
182
|
+
* over-engaged ones look like fraud and get flagged.
|
|
183
|
+
*
|
|
184
|
+
* -------------------------------------------------------------------
|
|
185
|
+
* 5. APPLICATION COMPLETION TIME-TO-CONVERT (everything)
|
|
186
|
+
* -------------------------------------------------------------------
|
|
187
|
+
*
|
|
188
|
+
* PATTERN: In the everything hook, place each "application approved"
|
|
189
|
+
* event at a fixed target offset from the first "application started"
|
|
190
|
+
* event, determined by account_type (assigned via deterministic hash):
|
|
191
|
+
* - business: ~36h target offset (~0.75x baseline)
|
|
192
|
+
* - individual: ~48h target offset (baseline)
|
|
193
|
+
* - family: ~63h target offset (~1.31x baseline)
|
|
194
|
+
*
|
|
195
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
196
|
+
*
|
|
197
|
+
* Report 1: Application Funnel TTC by Account Type
|
|
198
|
+
* - Report type: Funnels
|
|
199
|
+
* - Steps: "application started" → "application approved"
|
|
200
|
+
* - Breakdown: account_type (from account created event)
|
|
201
|
+
* - Measure: Median time to convert
|
|
202
|
+
* - Expected: business ~37h < individual ~50h < family ~65h medians
|
|
203
|
+
* (target + 0-4h jitter); ratios ~0.75x / 1.0x / ~1.30x
|
|
204
|
+
* - NOTE: account_type is stamped only on "account created" events,
|
|
205
|
+
* which only born-in-dataset users have — pre-existing users' gaps
|
|
206
|
+
* are engineered identically but show as (not set) in the breakdown.
|
|
207
|
+
*
|
|
208
|
+
* REAL-WORLD ANALOGUE: Business applicants have streamlined processes
|
|
209
|
+
* and pre-filled forms; family policies require more documentation.
|
|
210
|
+
*
|
|
211
|
+
* -------------------------------------------------------------------
|
|
212
|
+
* 6. CLAIMS PROCESS EXPERIMENT (funnel config — no hook code)
|
|
213
|
+
* -------------------------------------------------------------------
|
|
214
|
+
*
|
|
215
|
+
* PATTERN: A/B experiment on the Claims Process funnel. The engine
|
|
216
|
+
* fires $experiment_started events and applies conversion/TTC
|
|
217
|
+
* multipliers automatically. "Simplified Claims" variant gets 1.3x
|
|
218
|
+
* conversion and 0.8x time-to-convert. Experiment runs in the last
|
|
219
|
+
* 35 days of the dataset.
|
|
220
|
+
*
|
|
221
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
222
|
+
*
|
|
223
|
+
* Report 1: Experiment Results
|
|
224
|
+
* - Report type: Funnels
|
|
225
|
+
* - Steps: "claim filed" -> "claim status checked" -> "support ticket created"
|
|
226
|
+
* - Breakdown: "Variant name"
|
|
227
|
+
* - Date range: last 35 days of dataset
|
|
228
|
+
* - Expected: "Simplified Claims" has ~1.3x conversion vs "Control"
|
|
229
|
+
*
|
|
230
|
+
* Report 2: Experiment Started Events
|
|
231
|
+
* - Report type: Insights
|
|
232
|
+
* - Event: "$experiment_started"
|
|
233
|
+
* - Breakdown: "Variant name"
|
|
234
|
+
* - Expected: roughly equal distribution between Control and Simplified Claims
|
|
235
|
+
*
|
|
236
|
+
* REAL-WORLD ANALOGUE: A/B testing a simplified claims flow to reduce
|
|
237
|
+
* friction and improve completion rates.
|
|
238
|
+
*
|
|
239
|
+
* -------------------------------------------------------------------
|
|
240
|
+
* 7. RISK PROFILE AFFECTS APPROVAL (funnel-pre)
|
|
241
|
+
* -------------------------------------------------------------------
|
|
242
|
+
*
|
|
243
|
+
* PATTERN: In the funnel-pre hook, low-risk users get 1.8x approval
|
|
244
|
+
* funnel conversion (capped at 95%), high-risk users get 0.3x.
|
|
245
|
+
* Medium-risk users are unchanged.
|
|
246
|
+
*
|
|
247
|
+
* OBSERVABLE MAGNITUDE: the Application Approval funnel's base
|
|
248
|
+
* conversionRate is 70%, so per-instance rates land at
|
|
249
|
+
* low = min(95, round(70x1.8)) = 95%, medium = 70%,
|
|
250
|
+
* high = round(70x0.3) = 21%. But non-converting instances still take
|
|
251
|
+
* a uniform 1..(steps-1) partial walk (determineConversion,
|
|
252
|
+
* lib/generators/funnels.js:527-534), so step 2 ("application
|
|
253
|
+
* approved") fires with probability p = c + (1-c)/2 — 0.975 / 0.85 /
|
|
254
|
+
* 0.605 — compressing the visible approvals-per-submit ratios to
|
|
255
|
+
* ~1.15x (low/med) and ~0.71x (high/med) instead of the naive
|
|
256
|
+
* 1.8x / 0.3x knob values.
|
|
257
|
+
*
|
|
258
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
259
|
+
*
|
|
260
|
+
* Report 1: Application Approval Funnel by Risk Profile
|
|
261
|
+
* - Report type: Funnels
|
|
262
|
+
* - Steps: "application submitted" -> "application approved" -> "policy activated"
|
|
263
|
+
* - Breakdown: "risk_profile" (user property)
|
|
264
|
+
* - Expected: low >> medium >> high conversion rates
|
|
265
|
+
*
|
|
266
|
+
* REAL-WORLD ANALOGUE: Underwriting engines auto-approve low-risk
|
|
267
|
+
* applicants and require manual review for high-risk ones.
|
|
268
|
+
*
|
|
269
|
+
* -------------------------------------------------------------------
|
|
270
|
+
* 8. DOCUMENT UPLOAD RETENTION (everything — retention magic number)
|
|
271
|
+
* -------------------------------------------------------------------
|
|
272
|
+
*
|
|
273
|
+
* PATTERN: Users who upload 3+ documents in their first 14 days
|
|
274
|
+
* retain normally. Users with fewer uploads lose 75% of events after
|
|
275
|
+
* day 30. Only affects born-in-dataset users.
|
|
276
|
+
*
|
|
277
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
278
|
+
*
|
|
279
|
+
* Report 1: Retention by Document Upload Cohort
|
|
280
|
+
* - Report type: Retention
|
|
281
|
+
* - Starting event: "account created"
|
|
282
|
+
* - Return event: any event
|
|
283
|
+
* - Cohort A: users with >= 3 "document uploaded" in first 14 days
|
|
284
|
+
* - Cohort B: users with < 3 "document uploaded" in first 14 days
|
|
285
|
+
* - Expected: Cohort A retains well past day 30; Cohort B drops off sharply
|
|
286
|
+
*
|
|
287
|
+
* Report 2: Post-Day-30 Event Volume
|
|
288
|
+
* - Report type: Insights
|
|
289
|
+
* - Event: all events
|
|
290
|
+
* - Compare doc-uploader cohort vs non-uploader
|
|
291
|
+
* - Expected: non-uploaders have ~75% fewer events after day 30
|
|
292
|
+
*
|
|
293
|
+
* REAL-WORLD ANALOGUE: Users who complete onboarding paperwork early
|
|
294
|
+
* are invested in the product and retain; those who don't churn.
|
|
295
|
+
*
|
|
296
|
+
* -------------------------------------------------------------------
|
|
297
|
+
* 9. END-OF-QUARTER RENEWAL SPIKE (everything — temporal)
|
|
298
|
+
* -------------------------------------------------------------------
|
|
299
|
+
*
|
|
300
|
+
* PATTERN: Days 85-95 of the dataset get 3x renewal completed events
|
|
301
|
+
* and 2x coverage reviewed events via cloning. Simulates end-of-quarter
|
|
302
|
+
* policy renewal batch processing.
|
|
303
|
+
*
|
|
304
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
305
|
+
*
|
|
306
|
+
* Report 1: Renewal Volume Over Time
|
|
307
|
+
* - Report type: Insights
|
|
308
|
+
* - Event: "renewal completed"
|
|
309
|
+
* - Measure: Total
|
|
310
|
+
* - Line chart by day
|
|
311
|
+
* - Expected: visible spike at days 85-95, ~3x baseline volume
|
|
312
|
+
*
|
|
313
|
+
* Report 2: Coverage Review Spike
|
|
314
|
+
* - Report type: Insights
|
|
315
|
+
* - Event: "coverage reviewed"
|
|
316
|
+
* - Line chart by day
|
|
317
|
+
* - Expected: ~2x volume during days 85-95 (reads slightly under 2x
|
|
318
|
+
* against a local-day baseline — the Mar 27-Apr 5 window is
|
|
319
|
+
* weekend-heavy, depressing the organic in-window rate)
|
|
320
|
+
*
|
|
321
|
+
* REAL-WORLD ANALOGUE: Insurance companies batch-process renewals at
|
|
322
|
+
* quarter-end, producing predictable volume spikes.
|
|
323
|
+
*
|
|
324
|
+
* -------------------------------------------------------------------
|
|
325
|
+
* 10. CLAIM-TO-PREMIUM INCREASE (event — closure Map)
|
|
326
|
+
* -------------------------------------------------------------------
|
|
327
|
+
*
|
|
328
|
+
* PATTERN: After a user files a claim, their next "payment made" event
|
|
329
|
+
* gets premium_amount doubled (2.0x). Uses a module-level Map to
|
|
330
|
+
* track state across events. The Map entry is consumed (deleted) on
|
|
331
|
+
* the first payment after the claim, so the effect is one-shot.
|
|
332
|
+
*
|
|
333
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
334
|
+
*
|
|
335
|
+
* Report 1: Average Premium After Claim
|
|
336
|
+
* - Report type: Insights
|
|
337
|
+
* - Event: "payment made"
|
|
338
|
+
* - Measure: Average of "premium_amount"
|
|
339
|
+
* - Cohort A: users who did "claim filed" before
|
|
340
|
+
* - Cohort B: users who never filed a claim
|
|
341
|
+
* - Expected: each DOUBLED payment is 2.0x its organic value, but
|
|
342
|
+
* only the first payment after each claim doubles — with claims and
|
|
343
|
+
* payments interleaving, roughly half of a claimant's payments get
|
|
344
|
+
* doubled, so the cohort-average ratio reads ~1.5x, not 2.0x.
|
|
345
|
+
* - Signature: organic premium_amount caps at 600, so ANY payment
|
|
346
|
+
* above 600 is claim-inflated. Non-claimants have exactly zero
|
|
347
|
+
* payments above 600, up to two rare orphan paths where the claim
|
|
348
|
+
* is removed AFTER the event hook already consumed the Map:
|
|
349
|
+
* (a) Hook #8 churn deletion (born non-uploaders), excluded from
|
|
350
|
+
* the verification population, and (b) the engine's unconditional
|
|
351
|
+
* future-time guard — the event hook fires in GENERATION order,
|
|
352
|
+
* not time order, so a claim generated past dataset end doubles
|
|
353
|
+
* the next-generated payment and is then dropped by the guard
|
|
354
|
+
* (~1 user in 15K at full fidelity). Claimants have thousands.
|
|
355
|
+
*
|
|
356
|
+
* REAL-WORLD ANALOGUE: Insurance premiums increase after filing a claim.
|
|
357
|
+
*
|
|
358
|
+
* ===================================================================
|
|
359
|
+
* EXPECTED METRICS SUMMARY (mechanism → measured at full fidelity)
|
|
360
|
+
* ===================================================================
|
|
361
|
+
*
|
|
362
|
+
* Story id | Metric | Expected | Measured (15K users, 1.98M events)
|
|
363
|
+
* ---------|-------------------------------------------|---------------|---------
|
|
364
|
+
* H1 | events with wrong version for timestamp | 0 (exact) | 0 of 1,976,808
|
|
365
|
+
* H1 | device-only / null-uid events | 0 (exact) | 0 / 0 (born share 0.121)
|
|
366
|
+
* H2 | bug-category ticket share pre-v2.13 | ~0.67 | 0.669
|
|
367
|
+
* H2 | bug-category ticket share post-v2.13 | 0 (exact) | 0 of 2,660
|
|
368
|
+
* H2 | ticket rate v2.13 / last-10d-of-v2.12 | ~0.25-0.35 | 0.289
|
|
369
|
+
* H3 | activations/day post / pre release | ~20-30x | 26.5x
|
|
370
|
+
* H3 | submit→activate user conversion post/pre | ~9-16x | 11.2x (0.045 → 0.499)
|
|
371
|
+
* H4 | sweet(8-14) / base(0-7) avg premium | 1.35x | 1.339x
|
|
372
|
+
* H4 | over(15+) / sweet approvals-per-submit | ~0.5-0.6x | 0.523x
|
|
373
|
+
* H5 | TTC medians biz/indiv/family (hours) | ~37 / ~50 / ~65 | 37.8 / 49.8 / 64.8
|
|
374
|
+
* H5 | TTC ratios biz/indiv, family/indiv | ~0.75 / ~1.30 | 0.758 / 1.301
|
|
375
|
+
* H6 | variant split, pre-start exposures | ~50/50, 0 | 410/371 (0.475), 0
|
|
376
|
+
* H6 | simplified/control claims completion | ~1.3-1.4x | 1.382x
|
|
377
|
+
* H7 | low/med approvals-per-submit | ~1.15x | 1.166x
|
|
378
|
+
* H7 | high/med approvals-per-submit | ~0.71x | 0.697x
|
|
379
|
+
* H8 | churn DiD (nonup post/pre ÷ up post/pre) | ~0.25-0.30 | 0.321 (sep 3.11x; n=218/40)
|
|
380
|
+
* H9 | renewals in spike / local base | 3.0x | 3.059x
|
|
381
|
+
* H9 | coverage reviews in spike / local base | ~1.8-2.0x | 1.781x
|
|
382
|
+
* H10 | claimant / non-claimant avg premium | ~1.5x | 1.391x
|
|
383
|
+
* H10 | non-claimant payments > 600 (untouched) | 0 (exact) | 1 (future-time-guard orphan; ≤2 STRONG)
|
|
384
|
+
*/
|
|
385
|
+
|
|
386
|
+
// ── SCALE ──
|
|
387
|
+
const SEED = "dm4-insurance";
|
|
388
|
+
const NUM_USERS = 15_000;
|
|
389
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
390
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
391
|
+
const EVENTS_PER_DAY = 1.2;
|
|
392
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
393
|
+
|
|
394
|
+
const chance = u.initChance(SEED);
|
|
395
|
+
|
|
396
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
397
|
+
// H1: Version stamping cutoffs (days from datasetStart / before datasetEnd)
|
|
398
|
+
const V211_DAY = 30;
|
|
399
|
+
const V212_DAY = 60;
|
|
400
|
+
const V213_DAYS_BEFORE_END = 10;
|
|
401
|
+
|
|
402
|
+
// H2: Support ticket volume
|
|
403
|
+
const TICKET_INJECT_MIN = 2;
|
|
404
|
+
const TICKET_INJECT_MAX = 3;
|
|
405
|
+
const TICKET_REMOVAL_BASE_PCT = 30;
|
|
406
|
+
const TICKET_REMOVAL_PER_DAY_PCT = 5.5;
|
|
407
|
+
const TICKET_REMOVAL_CAP_PCT = 85;
|
|
408
|
+
|
|
409
|
+
// H3: Application conversion boost
|
|
410
|
+
const PRE_V213_APPROVAL_DROP_LIKELIHOOD = 95;
|
|
411
|
+
|
|
412
|
+
// H4: Application-step magic number
|
|
413
|
+
const STEP_SWEET_MIN = 8;
|
|
414
|
+
const STEP_SWEET_MAX = 14;
|
|
415
|
+
const STEP_OVER_THRESHOLD = 15;
|
|
416
|
+
const STEP_PREMIUM_BOOST = 1.35;
|
|
417
|
+
const STEP_OVER_DROP_LIKELIHOOD = 40;
|
|
418
|
+
|
|
419
|
+
// H5: Application TTC target offsets (hours from first application started)
|
|
420
|
+
const TTC_BUSINESS_HOURS = 36;
|
|
421
|
+
const TTC_INDIVIDUAL_HOURS = 48;
|
|
422
|
+
const TTC_FAMILY_HOURS = 63;
|
|
423
|
+
|
|
424
|
+
// H7: Risk profile approval multipliers
|
|
425
|
+
const RISK_LOW_CONV_MULT = 1.8;
|
|
426
|
+
const RISK_HIGH_CONV_MULT = 0.3;
|
|
427
|
+
const RISK_CONV_CAP = 95;
|
|
428
|
+
|
|
429
|
+
// H8: Document upload retention
|
|
430
|
+
const DOC_RETENTION_MIN = 3;
|
|
431
|
+
const DOC_RETENTION_WINDOW_DAYS = 14;
|
|
432
|
+
const DOC_RETENTION_CUTOFF_DAYS = 30;
|
|
433
|
+
const DOC_RETENTION_DROP_LIKELIHOOD = 75;
|
|
434
|
+
|
|
435
|
+
// H9: End-of-quarter renewal spike (days from datasetStart)
|
|
436
|
+
const RENEWAL_SPIKE_START_DAY = 85;
|
|
437
|
+
const RENEWAL_SPIKE_END_DAY = 95;
|
|
438
|
+
const RENEWAL_CLONE_COUNT = 2; // 2 extra clones per renewal → 3x volume
|
|
439
|
+
const COVERAGE_CLONE_COUNT = 1; // 1 extra clone per coverage review → 2x volume
|
|
440
|
+
|
|
441
|
+
// H10: Claim-to-premium increase
|
|
442
|
+
const POST_CLAIM_PREMIUM_MULT = 2.0;
|
|
443
|
+
|
|
444
|
+
// ── HOOK STATE ──
|
|
445
|
+
// H10 closure: tracks users who filed a claim (one-shot consumption)
|
|
446
|
+
const claimFilers = new Map();
|
|
447
|
+
|
|
448
|
+
// H2: issue_category split. Bug categories are part of the declared
|
|
449
|
+
// issue_category value list (schema rule: hooks only modify existing
|
|
450
|
+
// props), so organic tickets pick them ~4/9 of the time. Post-v2.13 the
|
|
451
|
+
// hook reassigns surviving tickets' bug categories to non-bug values —
|
|
452
|
+
// the release fixed the bugs, so bug-category share drops to exactly 0.
|
|
453
|
+
const BUG_CATEGORIES = ["form_crash", "login_error", "page_timeout", "payment_failure"];
|
|
454
|
+
const NONBUG_CATEGORIES = ["billing", "claims", "coverage", "technical", "policy_change"];
|
|
455
|
+
|
|
456
|
+
// ── HELPER FUNCTIONS ──
|
|
457
|
+
function handleFunnelPreHooks(record, meta) {
|
|
458
|
+
// H7: RISK PROFILE AFFECTS APPROVAL — low-risk 1.8x, high-risk 0.3x conversion
|
|
459
|
+
const isApprovalFunnel = meta.funnel?.sequence?.includes("application approved");
|
|
460
|
+
if (isApprovalFunnel) {
|
|
461
|
+
const risk = meta.profile?.risk_profile;
|
|
462
|
+
if (risk === "low") record.conversionRate = Math.min(RISK_CONV_CAP, Math.round(record.conversionRate * RISK_LOW_CONV_MULT));
|
|
463
|
+
else if (risk === "high") record.conversionRate = Math.round(record.conversionRate * RISK_HIGH_CONV_MULT);
|
|
464
|
+
}
|
|
465
|
+
return record;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function handleEventHooks(record) {
|
|
469
|
+
// H10: CLAIM-TO-PREMIUM INCREASE — premium doubled on first payment after claim
|
|
470
|
+
if (record.event === "claim filed") {
|
|
471
|
+
claimFilers.set(record.user_id, true);
|
|
472
|
+
}
|
|
473
|
+
if (record.event === "payment made" && claimFilers.has(record.user_id)) {
|
|
474
|
+
record.premium_amount = Math.round((record.premium_amount || 500) * POST_CLAIM_PREMIUM_MULT);
|
|
475
|
+
claimFilers.delete(record.user_id);
|
|
476
|
+
}
|
|
477
|
+
return record;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function handleEverythingHooks(record, meta) {
|
|
481
|
+
// UTC mode is load-bearing: dayjs.unix() returns a LOCAL-mode instance, and
|
|
482
|
+
// local .add(N, "days") does calendar-day arithmetic — it slips 1h across the
|
|
483
|
+
// March 8 2026 DST spring-forward, making every derived day boundary past
|
|
484
|
+
// early March (the H9 spike window, H8 per-user retention windows) land an
|
|
485
|
+
// hour off and — worse — depend on the host machine's timezone, breaking the
|
|
486
|
+
// seeded-determinism contract. All day arithmetic below must stay UTC-mode.
|
|
487
|
+
const datasetStart = dayjs.unix(meta.datasetStart).utc();
|
|
488
|
+
const datasetEnd = dayjs.unix(meta.datasetEnd).utc();
|
|
489
|
+
const V211_DATE = datasetStart.add(V211_DAY, "days");
|
|
490
|
+
const V212_DATE = datasetStart.add(V212_DAY, "days");
|
|
491
|
+
const V213_DATE = datasetEnd.subtract(V213_DAYS_BEFORE_END, "days");
|
|
492
|
+
const userEvents = record;
|
|
493
|
+
if (userEvents.length === 0) return record;
|
|
494
|
+
|
|
495
|
+
// Stamp superProps from profile for consistency
|
|
496
|
+
const profile = meta.profile;
|
|
497
|
+
userEvents.forEach(e => {
|
|
498
|
+
e.Platform = profile.Platform;
|
|
499
|
+
e.insurance_type = profile.insurance_type;
|
|
500
|
+
e.app_version = profile.app_version;
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
// Find a user_id from any existing event
|
|
504
|
+
const userId =
|
|
505
|
+
userEvents.find((e) => e.user_id)?.user_id ||
|
|
506
|
+
userEvents[0]?.device_id;
|
|
507
|
+
|
|
508
|
+
// ─── Hook #5: APPLICATION COMPLETION TIME-TO-CONVERT ───
|
|
509
|
+
// Runs FIRST: adjusts timestamps before version stamping.
|
|
510
|
+
// Business accounts complete the application funnel faster (0.74x),
|
|
511
|
+
// family accounts slower (1.3x). Individual stays at baseline.
|
|
512
|
+
// Assigns account_type deterministically per user via djb2 hash
|
|
513
|
+
// (independent of engine RNG). Places each "application approved"
|
|
514
|
+
// event at a fixed offset from the first "application started".
|
|
515
|
+
{
|
|
516
|
+
// djb2 hash of userId → deterministic cohort
|
|
517
|
+
let h = 5381;
|
|
518
|
+
for (let i = 0; i < userId.length; i++) {
|
|
519
|
+
h = ((h << 5) + h + userId.charCodeAt(i)) | 0;
|
|
520
|
+
}
|
|
521
|
+
const acctTypes = ["individual", "family", "business"];
|
|
522
|
+
const userAccountType = acctTypes[((h % 3) + 3) % 3];
|
|
523
|
+
// Stamp account created events with the deterministic type
|
|
524
|
+
for (const evt of userEvents) {
|
|
525
|
+
if (evt.event === "account created") {
|
|
526
|
+
evt.account_type = userAccountType;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
userEvents.sort((a, b) => new Date(a.time) - new Date(b.time));
|
|
530
|
+
// Collect all "application started" times
|
|
531
|
+
const startedTimes = [];
|
|
532
|
+
for (const evt of userEvents) {
|
|
533
|
+
if (evt.event === "application started") {
|
|
534
|
+
startedTimes.push(dayjs(evt.time).valueOf());
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (startedTimes.length > 0) {
|
|
538
|
+
const firstStartTime = startedTimes[0];
|
|
539
|
+
// Target hours: biz=36, indiv=48, family=63
|
|
540
|
+
// Ratios: biz/indiv≈0.75, family/indiv≈1.31
|
|
541
|
+
const targetHours = (
|
|
542
|
+
userAccountType === "business" ? TTC_BUSINESS_HOURS :
|
|
543
|
+
userAccountType === "family" ? TTC_FAMILY_HOURS :
|
|
544
|
+
TTC_INDIVIDUAL_HOURS
|
|
545
|
+
);
|
|
546
|
+
const targetMs = targetHours * 3600000;
|
|
547
|
+
for (const evt of userEvents) {
|
|
548
|
+
if (evt.event !== "application approved") continue;
|
|
549
|
+
const evtTime = dayjs(evt.time).valueOf();
|
|
550
|
+
const origGap = evtTime - firstStartTime;
|
|
551
|
+
// Small jitter from original gap (mod 4h) for variance
|
|
552
|
+
const jitter = origGap > 0 ? (origGap % (4 * 3600000)) : 0;
|
|
553
|
+
evt.time = dayjs(firstStartTime + targetMs + jitter).toISOString();
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// ─── Hook #2: SUPPORT TICKET VOLUME ───
|
|
559
|
+
// PRE-V2.13: Inject 2-3 extra support tickets with bug-related categories
|
|
560
|
+
const preV213Tickets = userEvents.filter(
|
|
561
|
+
(e) =>
|
|
562
|
+
e.event === "support ticket created" &&
|
|
563
|
+
dayjs(e.time).isBefore(V213_DATE)
|
|
564
|
+
);
|
|
565
|
+
|
|
566
|
+
if (preV213Tickets.length > 0) {
|
|
567
|
+
const extraCount = chance.integer({ min: TICKET_INJECT_MIN, max: TICKET_INJECT_MAX });
|
|
568
|
+
|
|
569
|
+
for (let i = 0; i < extraCount; i++) {
|
|
570
|
+
// Pick a random pre-v2.13 ticket to base timing on
|
|
571
|
+
const sourceTicket = chance.pickone(preV213Tickets);
|
|
572
|
+
const sourceTime = dayjs(sourceTicket.time);
|
|
573
|
+
// Offset by a few hours to days
|
|
574
|
+
const offsetHours = chance.integer({ min: 1, max: 72 });
|
|
575
|
+
let newTime = sourceTime.add(offsetHours, "hours");
|
|
576
|
+
// Ensure it stays before v2.13
|
|
577
|
+
if (newTime.isAfter(V213_DATE)) {
|
|
578
|
+
newTime = V213_DATE.subtract(
|
|
579
|
+
chance.integer({ min: 1, max: 48 }),
|
|
580
|
+
"hours"
|
|
581
|
+
);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Compute app_version for injected event
|
|
585
|
+
let injectedVersion;
|
|
586
|
+
if (newTime.isBefore(V211_DATE)) {
|
|
587
|
+
injectedVersion = "2.10";
|
|
588
|
+
} else if (newTime.isBefore(V212_DATE)) {
|
|
589
|
+
injectedVersion = "2.11";
|
|
590
|
+
} else {
|
|
591
|
+
injectedVersion = "2.12"; // always pre-v2.13
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
userEvents.push({
|
|
595
|
+
...sourceTicket,
|
|
596
|
+
time: newTime.toISOString(),
|
|
597
|
+
user_id: userId,
|
|
598
|
+
app_version: injectedVersion,
|
|
599
|
+
issue_category: chance.pickone(BUG_CATEGORIES),
|
|
600
|
+
priority: chance.pickone(["medium", "high", "high"]),
|
|
601
|
+
channel: chance.pickone([
|
|
602
|
+
"chat",
|
|
603
|
+
"phone",
|
|
604
|
+
"email",
|
|
605
|
+
"web_form",
|
|
606
|
+
]),
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// POST-V2.13: Remove support tickets with increasing probability
|
|
612
|
+
// Day 1 after release: ~30% removal
|
|
613
|
+
// Day 5: ~60% removal
|
|
614
|
+
// Day 10: ~85% removal
|
|
615
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
616
|
+
const evt = userEvents[i];
|
|
617
|
+
if (evt.event === "support ticket created") {
|
|
618
|
+
const evtTime = dayjs(evt.time);
|
|
619
|
+
if (evtTime.isAfter(V213_DATE)) {
|
|
620
|
+
const daysSinceRelease = evtTime.diff(
|
|
621
|
+
V213_DATE,
|
|
622
|
+
"days",
|
|
623
|
+
true
|
|
624
|
+
);
|
|
625
|
+
// Linear ramp: 30% base + 5.5% per day → ~85% at day 10
|
|
626
|
+
const removalLikelihood = Math.min(
|
|
627
|
+
TICKET_REMOVAL_CAP_PCT,
|
|
628
|
+
TICKET_REMOVAL_BASE_PCT + daysSinceRelease * TICKET_REMOVAL_PER_DAY_PCT
|
|
629
|
+
);
|
|
630
|
+
if (chance.bool({ likelihood: removalLikelihood })) {
|
|
631
|
+
userEvents.splice(i, 1);
|
|
632
|
+
} else if (BUG_CATEGORIES.includes(evt.issue_category)) {
|
|
633
|
+
// v2.13 fixed the bugs: surviving post-release tickets can't
|
|
634
|
+
// be about them. Reassign to a non-bug category so the
|
|
635
|
+
// pre/post issue_category breakdown shows bug categories
|
|
636
|
+
// vanishing to exactly zero (see HOOK STORIES #2).
|
|
637
|
+
evt.issue_category = chance.pickone(NONBUG_CATEGORIES);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// ─── Hook #3: APPLICATION CONVERSION BOOST (activation gap) ───
|
|
644
|
+
// PRE-V2.13: Remove policy activated events for ~95% of users (per-user
|
|
645
|
+
// gating, not per-event). Produces a visible submitted → activated funnel
|
|
646
|
+
// gap pre-v2.13 vs post-v2.13.
|
|
647
|
+
//
|
|
648
|
+
// Deliberately does NOT touch "application approved": Hook #5 pins every
|
|
649
|
+
// approved event to firstStart + 36-63h, which lands almost all of them
|
|
650
|
+
// pre-v2.13 in time regardless of when the funnel instance ran. Dropping
|
|
651
|
+
// relocated approvals here would (a) leave no post-v2.13 approvals to
|
|
652
|
+
// step up — inverting this story — and (b) starve Hook #5's TTC cohort.
|
|
653
|
+
// The era signal therefore lives on policy activated alone: "approvals
|
|
654
|
+
// went through, but the activation step was broken until v2.13".
|
|
655
|
+
if (chance.bool({ likelihood: PRE_V213_APPROVAL_DROP_LIKELIHOOD })) {
|
|
656
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
657
|
+
const evt = userEvents[i];
|
|
658
|
+
if (
|
|
659
|
+
evt.event === "policy activated" &&
|
|
660
|
+
dayjs(evt.time).isBefore(V213_DATE)
|
|
661
|
+
) {
|
|
662
|
+
userEvents.splice(i, 1);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
// Hook 4: APPLICATION-STEP MAGIC NUMBER (no flags)
|
|
668
|
+
// Sweet 8-14 application step completed → +35% on approved_premium
|
|
669
|
+
// for application approved events. Over 15+ → drop 40% of
|
|
670
|
+
// application approved events (fraud filter triggers).
|
|
671
|
+
const stepCount = userEvents.filter(e => e.event === "application step completed").length;
|
|
672
|
+
if (stepCount >= STEP_SWEET_MIN && stepCount <= STEP_SWEET_MAX) {
|
|
673
|
+
userEvents.forEach(e => {
|
|
674
|
+
if (e.event === "application approved" && typeof e.approved_premium === "number") {
|
|
675
|
+
e.approved_premium = Math.round(e.approved_premium * STEP_PREMIUM_BOOST);
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
} else if (stepCount >= STEP_OVER_THRESHOLD) {
|
|
679
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
680
|
+
if (userEvents[i].event === "application approved" && chance.bool({ likelihood: STEP_OVER_DROP_LIKELIHOOD })) {
|
|
681
|
+
userEvents.splice(i, 1);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// ─── Hook #8: DOCUMENT UPLOAD RETENTION ───
|
|
687
|
+
// Users with 3+ "document uploaded" in first 14 days retain;
|
|
688
|
+
// others lose 75% of events post-day-30.
|
|
689
|
+
if (meta.userIsBornInDataset) {
|
|
690
|
+
const firstT = userEvents[0]?.time;
|
|
691
|
+
if (firstT) {
|
|
692
|
+
// dayjs.utc: local-mode .add(N, "days") slips 1h across the March DST
|
|
693
|
+
// spring-forward for users whose window straddles it (see header note).
|
|
694
|
+
const window14 = dayjs.utc(firstT).add(DOC_RETENTION_WINDOW_DAYS, "days").toISOString();
|
|
695
|
+
const docUploads = userEvents.filter(
|
|
696
|
+
(e) => e.event === "document uploaded" && e.time <= window14
|
|
697
|
+
).length;
|
|
698
|
+
if (docUploads < DOC_RETENTION_MIN) {
|
|
699
|
+
const cutoff = dayjs.utc(firstT).add(DOC_RETENTION_CUTOFF_DAYS, "days");
|
|
700
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
701
|
+
if (
|
|
702
|
+
dayjs(userEvents[i].time).isAfter(cutoff) &&
|
|
703
|
+
chance.bool({ likelihood: DOC_RETENTION_DROP_LIKELIHOOD })
|
|
704
|
+
) {
|
|
705
|
+
userEvents.splice(i, 1);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// ─── Hook #9: END-OF-QUARTER RENEWAL SPIKE ───
|
|
713
|
+
// Days 85-95 get 3x renewal clones + 2x coverage reviewed clones
|
|
714
|
+
{
|
|
715
|
+
const spikeStart = datasetStart.add(RENEWAL_SPIKE_START_DAY, "days");
|
|
716
|
+
const spikeEnd = datasetStart.add(RENEWAL_SPIKE_END_DAY, "days");
|
|
717
|
+
const clones = [];
|
|
718
|
+
userEvents.forEach((e) => {
|
|
719
|
+
const t = dayjs(e.time);
|
|
720
|
+
if (t.isAfter(spikeStart) && t.isBefore(spikeEnd)) {
|
|
721
|
+
if (e.event === "renewal completed") {
|
|
722
|
+
for (let c = 0; c < RENEWAL_CLONE_COUNT; c++) {
|
|
723
|
+
clones.push({
|
|
724
|
+
...e,
|
|
725
|
+
time: t
|
|
726
|
+
.add(
|
|
727
|
+
chance.integer({ min: 5, max: 240 }),
|
|
728
|
+
"minutes"
|
|
729
|
+
)
|
|
730
|
+
.toISOString(),
|
|
731
|
+
insert_id: chance.guid(),
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (e.event === "coverage reviewed") {
|
|
736
|
+
for (let c = 0; c < COVERAGE_CLONE_COUNT; c++) {
|
|
737
|
+
clones.push({
|
|
738
|
+
...e,
|
|
739
|
+
time: t
|
|
740
|
+
.add(
|
|
741
|
+
chance.integer({ min: 5, max: 120 }),
|
|
742
|
+
"minutes"
|
|
743
|
+
)
|
|
744
|
+
.toISOString(),
|
|
745
|
+
insert_id: chance.guid(),
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
});
|
|
751
|
+
if (clones.length) userEvents.push(...clones);
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// ─── Hook #1: VERSION STAMPING ───
|
|
755
|
+
// Runs LAST: stamps app_version based on FINAL timestamps
|
|
756
|
+
// (after Hook #5 TTC scaling has adjusted event times).
|
|
757
|
+
// v2.10 (days 0-30) → v2.11 (30-60) → v2.12 (60-90) → v2.13 (last 10 days)
|
|
758
|
+
for (const evt of userEvents) {
|
|
759
|
+
const eventTime = dayjs(evt.time);
|
|
760
|
+
if (eventTime.isBefore(V211_DATE)) {
|
|
761
|
+
evt.app_version = "2.10";
|
|
762
|
+
} else if (eventTime.isBefore(V212_DATE)) {
|
|
763
|
+
evt.app_version = "2.11";
|
|
764
|
+
} else if (eventTime.isBefore(V213_DATE)) {
|
|
765
|
+
evt.app_version = "2.12";
|
|
766
|
+
} else {
|
|
767
|
+
evt.app_version = "2.13";
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
// Sort events by time after injection/removal/shifting
|
|
772
|
+
userEvents.sort(
|
|
773
|
+
(a, b) => new Date(a.time) - new Date(b.time)
|
|
774
|
+
);
|
|
775
|
+
|
|
776
|
+
return record;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
// ── CONFIG ──
|
|
780
|
+
/** @type {Config} */
|
|
781
|
+
const config = {
|
|
782
|
+
version: 2,
|
|
783
|
+
seed: SEED,
|
|
784
|
+
datasetStart: DATASET_START,
|
|
785
|
+
datasetEnd: DATASET_END,
|
|
786
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
787
|
+
numUsers: NUM_USERS,
|
|
788
|
+
format: "json",
|
|
789
|
+
gzip: true,
|
|
790
|
+
credentials: {
|
|
791
|
+
token,
|
|
792
|
+
},
|
|
793
|
+
switches: {
|
|
794
|
+
hasSessionIds: true,
|
|
795
|
+
alsoInferFunnels: false,
|
|
796
|
+
hasLocation: true,
|
|
797
|
+
hasAndroidDevices: true,
|
|
798
|
+
hasIOSDevices: true,
|
|
799
|
+
hasDesktopDevices: true,
|
|
800
|
+
hasBrowser: false,
|
|
801
|
+
hasCampaigns: false,
|
|
802
|
+
isAnonymous: false,
|
|
803
|
+
hasAdSpend: false,
|
|
804
|
+
hasAvatar: true,
|
|
805
|
+
},
|
|
806
|
+
identity: {
|
|
807
|
+
avgDevicePerUser: 2,
|
|
808
|
+
},
|
|
809
|
+
concurrency: 1,
|
|
810
|
+
writeToDisk: false,
|
|
811
|
+
|
|
812
|
+
scdProps: {
|
|
813
|
+
policy_status: {
|
|
814
|
+
values: ["pending", "active", "lapsed", "renewed"],
|
|
815
|
+
frequency: "month",
|
|
816
|
+
timing: "fuzzy",
|
|
817
|
+
max: 8
|
|
818
|
+
}
|
|
819
|
+
},
|
|
820
|
+
mirrorProps: {},
|
|
821
|
+
lookupTables: [],
|
|
822
|
+
|
|
823
|
+
events: [
|
|
824
|
+
{
|
|
825
|
+
event: "account created",
|
|
826
|
+
weight: 1,
|
|
827
|
+
isFirstEvent: true,
|
|
828
|
+
isAuthEvent: true,
|
|
829
|
+
properties: {
|
|
830
|
+
signup_source: ["web", "mobile", "agent_referral", "partner"],
|
|
831
|
+
account_type: ["individual", "family", "business"],
|
|
832
|
+
},
|
|
833
|
+
},
|
|
834
|
+
{
|
|
835
|
+
event: "page viewed",
|
|
836
|
+
weight: 10,
|
|
837
|
+
properties: {
|
|
838
|
+
page_name: [
|
|
839
|
+
"home",
|
|
840
|
+
"quotes",
|
|
841
|
+
"coverage_options",
|
|
842
|
+
"claims",
|
|
843
|
+
"faq",
|
|
844
|
+
"profile",
|
|
845
|
+
"payment",
|
|
846
|
+
"documents",
|
|
847
|
+
],
|
|
848
|
+
referrer: ["direct", "google", "email", "social_media"],
|
|
849
|
+
},
|
|
850
|
+
},
|
|
851
|
+
{
|
|
852
|
+
event: "quote requested",
|
|
853
|
+
weight: 5,
|
|
854
|
+
properties: {
|
|
855
|
+
coverage_level: ["basic", "standard", "premium"],
|
|
856
|
+
deductible: u.weighNumRange(250, 5000, 0.5, 1000),
|
|
857
|
+
},
|
|
858
|
+
},
|
|
859
|
+
{
|
|
860
|
+
event: "quote received",
|
|
861
|
+
weight: 4,
|
|
862
|
+
properties: {
|
|
863
|
+
monthly_premium: u.weighNumRange(30, 800, 0.3, 150),
|
|
864
|
+
coverage_amount: u.weighNumRange(10000, 500000, 0.3, 100000),
|
|
865
|
+
quote_comparison_count: u.weighNumRange(1, 5, 0.5, 2),
|
|
866
|
+
},
|
|
867
|
+
},
|
|
868
|
+
{
|
|
869
|
+
event: "application started",
|
|
870
|
+
weight: 4,
|
|
871
|
+
isStrictEvent: false,
|
|
872
|
+
properties: {
|
|
873
|
+
coverage_level: ["basic", "standard", "premium"],
|
|
874
|
+
estimated_premium: u.weighNumRange(30, 800, 0.3, 150),
|
|
875
|
+
},
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
event: "application step completed",
|
|
879
|
+
weight: 6,
|
|
880
|
+
isStrictEvent: false,
|
|
881
|
+
properties: {
|
|
882
|
+
step_name: [
|
|
883
|
+
"personal_info",
|
|
884
|
+
"coverage_selection",
|
|
885
|
+
"medical_history",
|
|
886
|
+
"vehicle_info",
|
|
887
|
+
"beneficiary",
|
|
888
|
+
"review",
|
|
889
|
+
],
|
|
890
|
+
step_number: u.weighNumRange(1, 6),
|
|
891
|
+
time_on_step_sec: u.weighNumRange(15, 600, 0.3, 90),
|
|
892
|
+
},
|
|
893
|
+
},
|
|
894
|
+
{
|
|
895
|
+
event: "document uploaded",
|
|
896
|
+
weight: 3,
|
|
897
|
+
isStrictEvent: false,
|
|
898
|
+
properties: {
|
|
899
|
+
document_type: [
|
|
900
|
+
"drivers_license",
|
|
901
|
+
"proof_of_address",
|
|
902
|
+
"vehicle_registration",
|
|
903
|
+
"medical_records",
|
|
904
|
+
"property_photos",
|
|
905
|
+
],
|
|
906
|
+
file_size_kb: u.weighNumRange(50, 5000, 0.5, 500),
|
|
907
|
+
},
|
|
908
|
+
},
|
|
909
|
+
{
|
|
910
|
+
event: "application submitted",
|
|
911
|
+
weight: 3,
|
|
912
|
+
properties: {
|
|
913
|
+
documents_attached: u.weighNumRange(1, 5, 0.5, 2),
|
|
914
|
+
application_time_min: u.weighNumRange(5, 60, 0.3, 15),
|
|
915
|
+
},
|
|
916
|
+
},
|
|
917
|
+
{
|
|
918
|
+
event: "application approved",
|
|
919
|
+
weight: 2,
|
|
920
|
+
isStrictEvent: false,
|
|
921
|
+
properties: {
|
|
922
|
+
approved_premium: u.weighNumRange(30, 800, 0.3, 150),
|
|
923
|
+
approval_time_hours: u.weighNumRange(1, 72, 0.5, 24),
|
|
924
|
+
},
|
|
925
|
+
},
|
|
926
|
+
{
|
|
927
|
+
event: "policy activated",
|
|
928
|
+
weight: 2,
|
|
929
|
+
isStrictEvent: false,
|
|
930
|
+
properties: {
|
|
931
|
+
policy_term_months: [6, 12, 12, 12, 24],
|
|
932
|
+
effective_date_offset_days: u.weighNumRange(0, 30, 0.5, 7),
|
|
933
|
+
},
|
|
934
|
+
},
|
|
935
|
+
{
|
|
936
|
+
event: "claim filed",
|
|
937
|
+
weight: 2,
|
|
938
|
+
isStrictEvent: false,
|
|
939
|
+
properties: {
|
|
940
|
+
claim_type: [
|
|
941
|
+
"collision",
|
|
942
|
+
"theft",
|
|
943
|
+
"water_damage",
|
|
944
|
+
"fire",
|
|
945
|
+
"medical",
|
|
946
|
+
"liability",
|
|
947
|
+
],
|
|
948
|
+
estimated_amount: u.weighNumRange(100, 50000, 0.3, 3000),
|
|
949
|
+
},
|
|
950
|
+
},
|
|
951
|
+
{
|
|
952
|
+
event: "claim status checked",
|
|
953
|
+
weight: 4,
|
|
954
|
+
properties: {
|
|
955
|
+
claim_status: [
|
|
956
|
+
"submitted",
|
|
957
|
+
"under_review",
|
|
958
|
+
"approved",
|
|
959
|
+
"denied",
|
|
960
|
+
"payment_pending",
|
|
961
|
+
],
|
|
962
|
+
days_since_filed: u.weighNumRange(1, 60, 0.5, 10),
|
|
963
|
+
},
|
|
964
|
+
},
|
|
965
|
+
{
|
|
966
|
+
event: "payment made",
|
|
967
|
+
weight: 5,
|
|
968
|
+
isStrictEvent: false,
|
|
969
|
+
properties: {
|
|
970
|
+
payment_method: ["credit_card", "bank_transfer", "auto_pay", "check"],
|
|
971
|
+
amount: u.weighNumRange(30, 800, 0.3, 150),
|
|
972
|
+
premium_amount: u.weighNumRange(50, 600, 0.3, 150),
|
|
973
|
+
payment_status: ["success", "success", "success", "success", "failed"],
|
|
974
|
+
},
|
|
975
|
+
},
|
|
976
|
+
{
|
|
977
|
+
event: "support ticket created",
|
|
978
|
+
weight: 4,
|
|
979
|
+
isStrictEvent: false,
|
|
980
|
+
properties: {
|
|
981
|
+
issue_category: [
|
|
982
|
+
"billing",
|
|
983
|
+
"claims",
|
|
984
|
+
"coverage",
|
|
985
|
+
"technical",
|
|
986
|
+
"policy_change",
|
|
987
|
+
"form_crash",
|
|
988
|
+
"login_error",
|
|
989
|
+
"page_timeout",
|
|
990
|
+
"payment_failure",
|
|
991
|
+
],
|
|
992
|
+
priority: ["low", "medium", "medium", "high"],
|
|
993
|
+
channel: ["chat", "phone", "email", "web_form"],
|
|
994
|
+
},
|
|
995
|
+
},
|
|
996
|
+
{
|
|
997
|
+
event: "support ticket resolved",
|
|
998
|
+
weight: 3,
|
|
999
|
+
properties: {
|
|
1000
|
+
resolution_type: [
|
|
1001
|
+
"self_service",
|
|
1002
|
+
"agent_assisted",
|
|
1003
|
+
"escalated",
|
|
1004
|
+
"auto_resolved",
|
|
1005
|
+
],
|
|
1006
|
+
satisfaction_score: u.weighNumRange(1, 5, 1, 4),
|
|
1007
|
+
resolution_time_hours: u.weighNumRange(0.5, 72, 0.3, 8),
|
|
1008
|
+
},
|
|
1009
|
+
},
|
|
1010
|
+
{
|
|
1011
|
+
event: "coverage reviewed",
|
|
1012
|
+
weight: 4,
|
|
1013
|
+
isStrictEvent: false,
|
|
1014
|
+
properties: {
|
|
1015
|
+
current_premium: u.weighNumRange(30, 800, 0.3, 150),
|
|
1016
|
+
coverage_adequate: [true, true, true, false],
|
|
1017
|
+
},
|
|
1018
|
+
},
|
|
1019
|
+
{
|
|
1020
|
+
event: "profile updated",
|
|
1021
|
+
weight: 2,
|
|
1022
|
+
properties: {
|
|
1023
|
+
field_updated: [
|
|
1024
|
+
"address",
|
|
1025
|
+
"phone",
|
|
1026
|
+
"email",
|
|
1027
|
+
"beneficiary",
|
|
1028
|
+
"payment_method",
|
|
1029
|
+
"vehicle_info",
|
|
1030
|
+
],
|
|
1031
|
+
},
|
|
1032
|
+
},
|
|
1033
|
+
{
|
|
1034
|
+
event: "renewal completed",
|
|
1035
|
+
weight: 2,
|
|
1036
|
+
isStrictEvent: false,
|
|
1037
|
+
properties: {
|
|
1038
|
+
renewal_premium: u.weighNumRange(30, 800, 0.3, 150),
|
|
1039
|
+
premium_change_pct: u.weighNumRange(-15, 20, 1, 3),
|
|
1040
|
+
auto_renewed: [true, true, false],
|
|
1041
|
+
},
|
|
1042
|
+
},
|
|
1043
|
+
],
|
|
1044
|
+
|
|
1045
|
+
funnels: [
|
|
1046
|
+
{
|
|
1047
|
+
name: "Onboarding",
|
|
1048
|
+
sequence: ["account created", "page viewed", "quote requested"],
|
|
1049
|
+
isFirstFunnel: true,
|
|
1050
|
+
conversionRate: 85,
|
|
1051
|
+
timeToConvert: 0.5,
|
|
1052
|
+
},
|
|
1053
|
+
{
|
|
1054
|
+
name: "Application Completion",
|
|
1055
|
+
sequence: [
|
|
1056
|
+
"application started",
|
|
1057
|
+
"application step completed",
|
|
1058
|
+
"document uploaded",
|
|
1059
|
+
"application submitted",
|
|
1060
|
+
],
|
|
1061
|
+
conversionRate: 60,
|
|
1062
|
+
timeToConvert: 48,
|
|
1063
|
+
weight: 4,
|
|
1064
|
+
order: "sequential",
|
|
1065
|
+
},
|
|
1066
|
+
{
|
|
1067
|
+
name: "Application Approval",
|
|
1068
|
+
sequence: [
|
|
1069
|
+
"application submitted",
|
|
1070
|
+
"application approved",
|
|
1071
|
+
"policy activated",
|
|
1072
|
+
],
|
|
1073
|
+
conversionRate: 70,
|
|
1074
|
+
timeToConvert: 72,
|
|
1075
|
+
weight: 3,
|
|
1076
|
+
order: "sequential",
|
|
1077
|
+
},
|
|
1078
|
+
{
|
|
1079
|
+
name: "Claims Process",
|
|
1080
|
+
sequence: [
|
|
1081
|
+
"claim filed",
|
|
1082
|
+
"claim status checked",
|
|
1083
|
+
"support ticket created",
|
|
1084
|
+
],
|
|
1085
|
+
conversionRate: 50,
|
|
1086
|
+
timeToConvert: 24,
|
|
1087
|
+
weight: 2,
|
|
1088
|
+
experiment: {
|
|
1089
|
+
name: "Simplified Claims Flow",
|
|
1090
|
+
variants: [
|
|
1091
|
+
{ name: "Control" },
|
|
1092
|
+
{ name: "Simplified Claims", conversionMultiplier: 1.3, ttcMultiplier: 0.8 },
|
|
1093
|
+
],
|
|
1094
|
+
startDaysBeforeEnd: 35,
|
|
1095
|
+
},
|
|
1096
|
+
},
|
|
1097
|
+
{
|
|
1098
|
+
name: "Policy Renewal",
|
|
1099
|
+
sequence: ["coverage reviewed", "payment made", "renewal completed"],
|
|
1100
|
+
conversionRate: 65,
|
|
1101
|
+
timeToConvert: 72,
|
|
1102
|
+
weight: 3,
|
|
1103
|
+
},
|
|
1104
|
+
],
|
|
1105
|
+
|
|
1106
|
+
superProps: {
|
|
1107
|
+
Platform: ["web", "ios", "android"],
|
|
1108
|
+
insurance_type: ["auto", "home", "life", "health", "renters"],
|
|
1109
|
+
app_version: ["2.10"],
|
|
1110
|
+
},
|
|
1111
|
+
|
|
1112
|
+
userProps: {
|
|
1113
|
+
Platform: ["web", "ios", "android"],
|
|
1114
|
+
insurance_type: ["auto", "home", "life", "health", "renters"],
|
|
1115
|
+
app_version: ["2.10"],
|
|
1116
|
+
age_range: ["18-25", "26-35", "36-45", "46-55", "56-65", "65+"],
|
|
1117
|
+
risk_profile: ["low", "medium", "high"],
|
|
1118
|
+
policy_count: u.weighNumRange(0, 5, 0.5, 1),
|
|
1119
|
+
lifetime_premium: u.weighNumRange(0, 50000, 0.3, 5000),
|
|
1120
|
+
preferred_contact: ["email", "phone", "app_notification"],
|
|
1121
|
+
},
|
|
1122
|
+
|
|
1123
|
+
hook(record, type, meta) {
|
|
1124
|
+
if (type === "funnel-pre") return handleFunnelPreHooks(record, meta);
|
|
1125
|
+
if (type === "event") return handleEventHooks(record);
|
|
1126
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
1127
|
+
return record;
|
|
1128
|
+
},
|
|
1129
|
+
};
|
|
1130
|
+
|
|
1131
|
+
export default config;
|
|
1132
|
+
|
|
1133
|
+
// ── STORIES (machine-checkable contract; evaluate with scripts/verify-stories.mjs) ──
|
|
1134
|
+
/*
|
|
1135
|
+
* STORY DOCTRINE (insurance-application)
|
|
1136
|
+
*
|
|
1137
|
+
* LITERAL WINDOW: this dungeon pins epochStart/epochEnd to literal dates
|
|
1138
|
+
* (2026-01-01 → 2026-05-01T23:59:59Z) with no forward shift, so all
|
|
1139
|
+
* day-index arithmetic runs off DATE '2026-01-01' (day_idx 0-120) and
|
|
1140
|
+
* version boundaries are exact timestamps:
|
|
1141
|
+
* v2.11 @ 2026-01-31 00:00:00 (start + 30d)
|
|
1142
|
+
* v2.12 @ 2026-03-02 00:00:00 (start + 60d)
|
|
1143
|
+
* v2.13 @ 2026-04-21 23:59:59 (end − 10d — end is 23:59:59, so this
|
|
1144
|
+
* boundary is NOT midnight; every era
|
|
1145
|
+
* split below uses the exact timestamp)
|
|
1146
|
+
* Pre-era = 111.0 days, post-era = 10.0 days, exactly.
|
|
1147
|
+
*
|
|
1148
|
+
* IDENTITY: "account created" is isFirstEvent + isAuthEvent, so identity
|
|
1149
|
+
* stitches on the very first event — every event carries user_id and
|
|
1150
|
+
* there are ZERO device-only events (asserted). avgDevicePerUser: 2 only
|
|
1151
|
+
* populates profile device pools ("anonymousIds").
|
|
1152
|
+
*
|
|
1153
|
+
* POPULATION RESTRICTION: H8 deletes 75% of post-day-30 events for
|
|
1154
|
+
* born-in non-uploaders, corrupting per-user counts for that cohort.
|
|
1155
|
+
* Reads that depend on undistorted per-user tallies (H4 step buckets,
|
|
1156
|
+
* H10 exact signature) restrict to the H8-untouched population:
|
|
1157
|
+
* (NOT born) OR uploader.
|
|
1158
|
+
*
|
|
1159
|
+
* H5 RELOCATION CAVEAT: H5 pins every "application approved" to
|
|
1160
|
+
* firstStart + {36,48,63}h (+0-4h jitter), so approvals cluster near
|
|
1161
|
+
* each user's start regardless of funnel-instance timing. Any read that
|
|
1162
|
+
* assumes approvals are era-distributed must account for this (H3
|
|
1163
|
+
* deliberately keys its era signal on "policy activated" instead).
|
|
1164
|
+
*
|
|
1165
|
+
* BAND DOCTRINE: NAILED bands center on mechanism numbers derived from
|
|
1166
|
+
* the knobs (with derivations in each narrative); reduced-scale
|
|
1167
|
+
* measurements (2K users, iter-ins-2) confirm placement but do not
|
|
1168
|
+
* define the centers. Cohort guards sized for full fidelity (15K users,
|
|
1169
|
+
* ~2M events) — reduced-scale runs hit scale-cap WEAKs on H6/H8.
|
|
1170
|
+
*/
|
|
1171
|
+
|
|
1172
|
+
const STORY_START_DATE = "2026-01-01";
|
|
1173
|
+
// Exact era boundaries (see doctrine): derived from DATASET_START/END + knobs.
|
|
1174
|
+
const V213_TS = "2026-04-21 23:59:59"; // end (2026-05-01 23:59:59) − V213_DAYS_BEFORE_END
|
|
1175
|
+
const EXP_START_TS = "2026-03-27 23:59:59"; // end − 35d (config-validator normalizeExperiments: startUnix = endUnix − startDays·86400)
|
|
1176
|
+
const PRE_ERA_DAYS = 111.0;
|
|
1177
|
+
const POST_ERA_DAYS = 10.0;
|
|
1178
|
+
|
|
1179
|
+
const ID_CTE = `
|
|
1180
|
+
us AS (SELECT distinct_id::VARCHAR AS duid, risk_profile
|
|
1181
|
+
FROM read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)),
|
|
1182
|
+
ev AS (SELECT e.user_id::VARCHAR AS uid, e.device_id::VARCHAR AS did, e.time::TIMESTAMP AS t,
|
|
1183
|
+
date_diff('day', DATE '${STORY_START_DATE}', e.time::TIMESTAMP::DATE) AS day_idx, e.*
|
|
1184
|
+
FROM read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true) e)`;
|
|
1185
|
+
|
|
1186
|
+
const PU_CTE = `
|
|
1187
|
+
pu AS (SELECT uid, min(t) AS first_t,
|
|
1188
|
+
count(*) FILTER (WHERE event = 'account created') AS acct_created,
|
|
1189
|
+
count(*) FILTER (WHERE event = 'application step completed') AS steps,
|
|
1190
|
+
count(*) FILTER (WHERE event = 'application approved') AS approvals,
|
|
1191
|
+
count(*) FILTER (WHERE event = 'application submitted') AS submits,
|
|
1192
|
+
count(*) FILTER (WHERE event = 'claim filed') AS claims,
|
|
1193
|
+
count(*) FILTER (WHERE event = 'payment made') AS payments
|
|
1194
|
+
FROM ev GROUP BY 1),
|
|
1195
|
+
puu AS (SELECT p.*, p.acct_created > 0 AS born,
|
|
1196
|
+
((SELECT count(*) FROM ev e WHERE e.uid = p.uid AND e.event = 'document uploaded'
|
|
1197
|
+
AND e.t <= p.first_t + INTERVAL '${DOC_RETENTION_WINDOW_DAYS} days') >= ${DOC_RETENTION_MIN}) AS uploader
|
|
1198
|
+
FROM pu p),
|
|
1199
|
+
h4pop AS (SELECT * FROM puu WHERE (NOT born) OR uploader)`;
|
|
1200
|
+
|
|
1201
|
+
const cellsOf = (rows, key) => Object.fromEntries((rows || []).map((r) => [r[key], r]));
|
|
1202
|
+
|
|
1203
|
+
export const stories = [
|
|
1204
|
+
{
|
|
1205
|
+
id: "H1-version-bands",
|
|
1206
|
+
hook: "H1",
|
|
1207
|
+
archetype: "composition-drift",
|
|
1208
|
+
narrative:
|
|
1209
|
+
"Every event's app_version is a pure function of its timestamp (v2.10/2.11/2.12/2.13 " +
|
|
1210
|
+
"bands). Exact invariant: ZERO events whose version disagrees with their timestamp's " +
|
|
1211
|
+
"band — including the non-midnight v2.13 boundary at 2026-04-21 23:59:59 (t >= boundary " +
|
|
1212
|
+
"stamps 2.13, matching the hook's comparison). Also carries the identity invariants: " +
|
|
1213
|
+
"auth-on-first-event means user_id on every event, no device-only rows, exactly one " +
|
|
1214
|
+
"account created per born user.",
|
|
1215
|
+
assertions: [
|
|
1216
|
+
{
|
|
1217
|
+
breakdown: {
|
|
1218
|
+
type: "duckdb",
|
|
1219
|
+
sql: `WITH ${ID_CTE}
|
|
1220
|
+
SELECT count(*) FILTER (WHERE app_version != CASE
|
|
1221
|
+
WHEN t < TIMESTAMP '2026-01-31 00:00:00' THEN '2.10'
|
|
1222
|
+
WHEN t < TIMESTAMP '2026-03-02 00:00:00' THEN '2.11'
|
|
1223
|
+
WHEN t < TIMESTAMP '${V213_TS}' THEN '2.12'
|
|
1224
|
+
ELSE '2.13' END)::BIGINT AS violations,
|
|
1225
|
+
count(DISTINCT app_version)::BIGINT AS versions,
|
|
1226
|
+
count(*) FILTER (WHERE app_version = '2.13')::BIGINT AS v213_n,
|
|
1227
|
+
count(*)::BIGINT AS total
|
|
1228
|
+
FROM ev`,
|
|
1229
|
+
},
|
|
1230
|
+
assert: (rows) => {
|
|
1231
|
+
const r = rows?.[0];
|
|
1232
|
+
if (!r || Number(r.total) < 100000) {
|
|
1233
|
+
return { verdict: "WEAK", detail: `event volume too small: total=${r?.total ?? 0}` };
|
|
1234
|
+
}
|
|
1235
|
+
const v = Number(r.violations);
|
|
1236
|
+
const detail = `version/timestamp violations=${v} of ${r.total} (versions=${r.versions}, v2.13 n=${r.v213_n})`;
|
|
1237
|
+
if (v === 0 && Number(r.versions) === 4 && Number(r.v213_n) > 0) return { verdict: "NAILED", detail };
|
|
1238
|
+
if (v <= Number(r.total) * 0.0005) return { verdict: "STRONG", detail };
|
|
1239
|
+
if (v <= Number(r.total) * 0.01) return { verdict: "WEAK", detail };
|
|
1240
|
+
return { verdict: "INVERSE", detail };
|
|
1241
|
+
},
|
|
1242
|
+
},
|
|
1243
|
+
{
|
|
1244
|
+
breakdown: {
|
|
1245
|
+
type: "duckdb",
|
|
1246
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1247
|
+
SELECT (SELECT count(*) FROM ev WHERE uid IS NULL)::BIGINT AS null_uid,
|
|
1248
|
+
(SELECT count(*) FROM ev WHERE uid IS NULL AND did IS NOT NULL)::BIGINT AS device_only,
|
|
1249
|
+
(SELECT count(*) FROM pu WHERE acct_created > 1)::BIGINT AS multi_acct,
|
|
1250
|
+
(SELECT count(*) FROM pu WHERE acct_created > 0)::BIGINT AS born_users,
|
|
1251
|
+
(SELECT count(*) FROM pu)::BIGINT AS total_users`,
|
|
1252
|
+
},
|
|
1253
|
+
assert: (rows) => {
|
|
1254
|
+
const r = rows?.[0];
|
|
1255
|
+
if (!r || Number(r.total_users) < 1500) {
|
|
1256
|
+
return { verdict: "WEAK", detail: `user volume too small: total_users=${r?.total_users ?? 0}` };
|
|
1257
|
+
}
|
|
1258
|
+
const share = Number(r.born_users) / Number(r.total_users);
|
|
1259
|
+
const clean = Number(r.null_uid) === 0 && Number(r.device_only) === 0 && Number(r.multi_acct) === 0;
|
|
1260
|
+
const detail = `null_uid=${r.null_uid} device_only=${r.device_only} multi_acct=${r.multi_acct} born_share=${share.toFixed(3)} (${r.born_users}/${r.total_users})`;
|
|
1261
|
+
if (clean && share >= 0.08 && share <= 0.16) return { verdict: "NAILED", detail };
|
|
1262
|
+
if (clean && share >= 0.06 && share <= 0.2) return { verdict: "STRONG", detail };
|
|
1263
|
+
if (clean) return { verdict: "WEAK", detail };
|
|
1264
|
+
return { verdict: "INVERSE", detail };
|
|
1265
|
+
},
|
|
1266
|
+
},
|
|
1267
|
+
],
|
|
1268
|
+
},
|
|
1269
|
+
{
|
|
1270
|
+
id: "H2-ticket-drop",
|
|
1271
|
+
hook: "H2",
|
|
1272
|
+
archetype: "temporal-inflection",
|
|
1273
|
+
narrative:
|
|
1274
|
+
"Pre-v2.13 all-bug injections (2-3/user) lift bug-category share from the organic 4/9 " +
|
|
1275
|
+
"(0.444, uniform 9-value pool) to (0.444·O + I)/(O + I) ≈ 0.67 at I/O ≈ 0.7. Post-v2.13 " +
|
|
1276
|
+
"the hook removes tickets on a 30%→85% ramp AND recategorizes surviving bug-category " +
|
|
1277
|
+
"tickets, so post bug share is EXACTLY ZERO. Volume: keep-rate avg ≈ 0.43 over the ramp " +
|
|
1278
|
+
"÷ ≈1.7x injection inflation ≈ 0.25 mechanism; organic late-dataset drift lifts the " +
|
|
1279
|
+
"observed ratio (0.31 measured at 2K).",
|
|
1280
|
+
assertions: [
|
|
1281
|
+
{
|
|
1282
|
+
breakdown: {
|
|
1283
|
+
type: "duckdb",
|
|
1284
|
+
sql: `WITH ${ID_CTE}
|
|
1285
|
+
SELECT count(*) FILTER (WHERE issue_category IN ('form_crash','login_error','page_timeout','payment_failure')
|
|
1286
|
+
AND t >= TIMESTAMP '${V213_TS}')::BIGINT AS bug_post_n,
|
|
1287
|
+
avg((issue_category IN ('form_crash','login_error','page_timeout','payment_failure'))::INT)
|
|
1288
|
+
FILTER (WHERE t < TIMESTAMP '${V213_TS}') AS bug_share_pre,
|
|
1289
|
+
count(*) FILTER (WHERE t < TIMESTAMP '${V213_TS}')::BIGINT AS n_pre,
|
|
1290
|
+
count(*) FILTER (WHERE t >= TIMESTAMP '${V213_TS}')::BIGINT AS n_post
|
|
1291
|
+
FROM ev WHERE event = 'support ticket created'`,
|
|
1292
|
+
},
|
|
1293
|
+
assert: (rows) => {
|
|
1294
|
+
const r = rows?.[0];
|
|
1295
|
+
if (!r || Number(r.n_pre) < 8000 || Number(r.n_post) < 250) {
|
|
1296
|
+
return { verdict: "WEAK", detail: `ticket volume too small: pre=${r?.n_pre ?? 0} post=${r?.n_post ?? 0}` };
|
|
1297
|
+
}
|
|
1298
|
+
const pre = Number(r.bug_share_pre), post = Number(r.bug_post_n);
|
|
1299
|
+
const detail = `bug share pre=${pre.toFixed(3)} (mechanism ~0.67), post bug tickets=${post} of ${r.n_post} (expect exactly 0)`;
|
|
1300
|
+
if (post === 0 && pre >= 0.62 && pre <= 0.72) return { verdict: "NAILED", detail };
|
|
1301
|
+
if (post <= 2 && pre >= 0.55 && pre <= 0.78) return { verdict: "STRONG", detail };
|
|
1302
|
+
if (pre > 0.5 && post < Number(r.n_post) * 0.2) return { verdict: "WEAK", detail };
|
|
1303
|
+
return { verdict: "INVERSE", detail };
|
|
1304
|
+
},
|
|
1305
|
+
},
|
|
1306
|
+
{
|
|
1307
|
+
breakdown: {
|
|
1308
|
+
type: "duckdb",
|
|
1309
|
+
sql: `WITH ${ID_CTE}
|
|
1310
|
+
SELECT count(*) FILTER (WHERE day_idx BETWEEN 101 AND 110)::BIGINT AS n_last10,
|
|
1311
|
+
count(*) FILTER (WHERE day_idx BETWEEN 111 AND 120)::BIGINT AS n_v213
|
|
1312
|
+
FROM ev WHERE event = 'support ticket created'`,
|
|
1313
|
+
},
|
|
1314
|
+
assert: (rows) => {
|
|
1315
|
+
const r = rows?.[0];
|
|
1316
|
+
if (!r || Number(r.n_last10) < 700 || Number(r.n_v213) < 200) {
|
|
1317
|
+
return { verdict: "WEAK", detail: `ticket volume too small: last10=${r?.n_last10 ?? 0} v213=${r?.n_v213 ?? 0}` };
|
|
1318
|
+
}
|
|
1319
|
+
const ratio = Number(r.n_v213) / Number(r.n_last10);
|
|
1320
|
+
const detail = `ticket rate v2.13/last-10d-of-v2.12=${ratio.toFixed(3)} (mechanism ~0.25 keep÷inflation; n=${r.n_v213}/${r.n_last10})`;
|
|
1321
|
+
if (ratio >= 0.22 && ratio <= 0.4) return { verdict: "NAILED", detail };
|
|
1322
|
+
if (ratio >= 0.16 && ratio <= 0.5) return { verdict: "STRONG", detail };
|
|
1323
|
+
if (ratio < 0.7) return { verdict: "WEAK", detail };
|
|
1324
|
+
return { verdict: "INVERSE", detail };
|
|
1325
|
+
},
|
|
1326
|
+
},
|
|
1327
|
+
],
|
|
1328
|
+
},
|
|
1329
|
+
{
|
|
1330
|
+
id: "H3-activation-gap",
|
|
1331
|
+
hook: "H3",
|
|
1332
|
+
archetype: "temporal-inflection",
|
|
1333
|
+
narrative:
|
|
1334
|
+
"Pre-v2.13, 95% of users lose ALL pre-release 'policy activated' events; approvals are " +
|
|
1335
|
+
"untouched (H5 pins them near each user's start — see doctrine). Volume mechanism: " +
|
|
1336
|
+
"1/0.05 = 20x step-up, amplified by organic late-dataset drift (submits drift ~1.13x " +
|
|
1337
|
+
"over the final 20 days; whole-era average sits further below the end-state) → ~28x " +
|
|
1338
|
+
"measured. Per-user era conversion (submit→activate within era) compresses below 20x " +
|
|
1339
|
+
"because the 111-day pre-era gives surviving activations far more exposure than the " +
|
|
1340
|
+
"10-day post-era: ~4% pre vs ~50% post ≈ 12x.",
|
|
1341
|
+
assertions: [
|
|
1342
|
+
{
|
|
1343
|
+
breakdown: {
|
|
1344
|
+
type: "duckdb",
|
|
1345
|
+
sql: `WITH ${ID_CTE}
|
|
1346
|
+
SELECT count(*) FILTER (WHERE t < TIMESTAMP '${V213_TS}')::BIGINT AS pre_n,
|
|
1347
|
+
count(*) FILTER (WHERE t >= TIMESTAMP '${V213_TS}')::BIGINT AS post_n
|
|
1348
|
+
FROM ev WHERE event = 'policy activated'`,
|
|
1349
|
+
},
|
|
1350
|
+
assert: (rows) => {
|
|
1351
|
+
const r = rows?.[0];
|
|
1352
|
+
if (!r || Number(r.post_n) < 800 || Number(r.pre_n) < 300) {
|
|
1353
|
+
return { verdict: "WEAK", detail: `activation volume too small: pre=${r?.pre_n ?? 0} post=${r?.post_n ?? 0}` };
|
|
1354
|
+
}
|
|
1355
|
+
const ratio = (Number(r.post_n) / POST_ERA_DAYS) / (Number(r.pre_n) / PRE_ERA_DAYS);
|
|
1356
|
+
const detail = `activations/day post/pre=${ratio.toFixed(1)}x (mechanism 20x gate x organic drift; n=${r.post_n}/${r.pre_n})`;
|
|
1357
|
+
if (ratio >= 20 && ratio <= 36) return { verdict: "NAILED", detail };
|
|
1358
|
+
if (ratio >= 14 && ratio <= 48) return { verdict: "STRONG", detail };
|
|
1359
|
+
if (ratio >= 5) return { verdict: "WEAK", detail };
|
|
1360
|
+
return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
|
|
1361
|
+
},
|
|
1362
|
+
},
|
|
1363
|
+
{
|
|
1364
|
+
breakdown: {
|
|
1365
|
+
type: "duckdb",
|
|
1366
|
+
sql: `WITH ${ID_CTE}
|
|
1367
|
+
SELECT
|
|
1368
|
+
(SELECT avg((EXISTS (SELECT 1 FROM ev a WHERE a.uid = s.uid AND a.event = 'policy activated' AND a.t < TIMESTAMP '${V213_TS}'))::INT)
|
|
1369
|
+
FROM (SELECT DISTINCT uid FROM ev WHERE event = 'application submitted' AND t < TIMESTAMP '${V213_TS}') s) AS pre_conv,
|
|
1370
|
+
(SELECT avg((EXISTS (SELECT 1 FROM ev a WHERE a.uid = s.uid AND a.event = 'policy activated' AND a.t >= TIMESTAMP '${V213_TS}'))::INT)
|
|
1371
|
+
FROM (SELECT DISTINCT uid FROM ev WHERE event = 'application submitted' AND t >= TIMESTAMP '${V213_TS}') s) AS post_conv,
|
|
1372
|
+
(SELECT count(DISTINCT uid) FROM ev WHERE event = 'application submitted' AND t < TIMESTAMP '${V213_TS}')::BIGINT AS pre_submitters,
|
|
1373
|
+
(SELECT count(DISTINCT uid) FROM ev WHERE event = 'application submitted' AND t >= TIMESTAMP '${V213_TS}')::BIGINT AS post_submitters`,
|
|
1374
|
+
},
|
|
1375
|
+
assert: (rows) => {
|
|
1376
|
+
const r = rows?.[0];
|
|
1377
|
+
if (!r || Number(r.pre_submitters) < 1200 || Number(r.post_submitters) < 1000) {
|
|
1378
|
+
return { verdict: "WEAK", detail: `submitter cohorts too small: pre=${r?.pre_submitters ?? 0} post=${r?.post_submitters ?? 0}` };
|
|
1379
|
+
}
|
|
1380
|
+
const pre = Number(r.pre_conv), post = Number(r.post_conv);
|
|
1381
|
+
const ratio = post / pre;
|
|
1382
|
+
const detail = `submit→activate user conversion pre=${pre.toFixed(3)} post=${post.toFixed(3)} ratio=${ratio.toFixed(1)}x (era-window compressed; n=${r.pre_submitters}/${r.post_submitters})`;
|
|
1383
|
+
if (ratio >= 9 && ratio <= 16) return { verdict: "NAILED", detail };
|
|
1384
|
+
if (ratio >= 6 && ratio <= 22) return { verdict: "STRONG", detail };
|
|
1385
|
+
if (ratio >= 3) return { verdict: "WEAK", detail };
|
|
1386
|
+
return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
|
|
1387
|
+
},
|
|
1388
|
+
},
|
|
1389
|
+
],
|
|
1390
|
+
},
|
|
1391
|
+
{
|
|
1392
|
+
id: "H4-step-magic",
|
|
1393
|
+
hook: "H4",
|
|
1394
|
+
archetype: "frequency-sweet-spot",
|
|
1395
|
+
narrative:
|
|
1396
|
+
"Sweet-spot users (8-14 'application step completed') get approved_premium x1.35; " +
|
|
1397
|
+
"over-engaged (15+) lose 40% of approvals. Population restricted to H8-untouched users " +
|
|
1398
|
+
"(doctrine) so step tallies are undistorted. The over-drop read normalizes approvals by " +
|
|
1399
|
+
"submitted applications — raw per-user approvals are activity-confounded (15+ steppers " +
|
|
1400
|
+
"organically earn ~2x approvals, measured raw ratio 0.77 vs normalized 0.53). Normalized " +
|
|
1401
|
+
"mechanism: 0.6 x selection factor ~0.8-1.0 (heavy steppers' submit mix skews toward " +
|
|
1402
|
+
"Application Completion conversions, inflating their denominator).",
|
|
1403
|
+
assertions: [
|
|
1404
|
+
{
|
|
1405
|
+
breakdown: {
|
|
1406
|
+
type: "duckdb",
|
|
1407
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1408
|
+
SELECT
|
|
1409
|
+
(SELECT avg(e.approved_premium) FROM ev e JOIN h4pop p ON e.uid = p.uid
|
|
1410
|
+
WHERE e.event = 'application approved' AND p.steps BETWEEN ${STEP_SWEET_MIN} AND ${STEP_SWEET_MAX}) AS sweet_avg,
|
|
1411
|
+
(SELECT avg(e.approved_premium) FROM ev e JOIN h4pop p ON e.uid = p.uid
|
|
1412
|
+
WHERE e.event = 'application approved' AND p.steps < ${STEP_SWEET_MIN}) AS base_avg,
|
|
1413
|
+
(SELECT count(*) FROM h4pop WHERE steps BETWEEN ${STEP_SWEET_MIN} AND ${STEP_SWEET_MAX})::BIGINT AS sweet_users,
|
|
1414
|
+
(SELECT count(*) FROM h4pop WHERE steps < ${STEP_SWEET_MIN})::BIGINT AS base_users`,
|
|
1415
|
+
},
|
|
1416
|
+
assert: (rows) => {
|
|
1417
|
+
const r = rows?.[0];
|
|
1418
|
+
if (!r || Number(r.sweet_users) < 500 || Number(r.base_users) < 140) {
|
|
1419
|
+
return { verdict: "WEAK", detail: `step cohorts too small: sweet=${r?.sweet_users ?? 0} base=${r?.base_users ?? 0}` };
|
|
1420
|
+
}
|
|
1421
|
+
const ratio = Number(r.sweet_avg) / Number(r.base_avg);
|
|
1422
|
+
const detail = `sweet/base avg approved_premium=${ratio.toFixed(3)} (knob ${STEP_PREMIUM_BOOST}x; users=${r.sweet_users}/${r.base_users})`;
|
|
1423
|
+
if (ratio >= 1.27 && ratio <= 1.43) return { verdict: "NAILED", detail };
|
|
1424
|
+
if (ratio >= 1.18 && ratio <= 1.52) return { verdict: "STRONG", detail };
|
|
1425
|
+
if (ratio >= 1.08) return { verdict: "WEAK", detail };
|
|
1426
|
+
return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
|
|
1427
|
+
},
|
|
1428
|
+
},
|
|
1429
|
+
{
|
|
1430
|
+
breakdown: {
|
|
1431
|
+
type: "duckdb",
|
|
1432
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1433
|
+
SELECT
|
|
1434
|
+
(SELECT sum(approvals)::DOUBLE / nullif(sum(submits), 0) FROM h4pop WHERE steps >= ${STEP_OVER_THRESHOLD}) AS over_aps,
|
|
1435
|
+
(SELECT sum(approvals)::DOUBLE / nullif(sum(submits), 0) FROM h4pop WHERE steps BETWEEN ${STEP_SWEET_MIN} AND ${STEP_SWEET_MAX}) AS sweet_aps,
|
|
1436
|
+
(SELECT sum(submits) FROM h4pop WHERE steps >= ${STEP_OVER_THRESHOLD})::BIGINT AS over_submits,
|
|
1437
|
+
(SELECT sum(submits) FROM h4pop WHERE steps BETWEEN ${STEP_SWEET_MIN} AND ${STEP_SWEET_MAX})::BIGINT AS sweet_submits`,
|
|
1438
|
+
},
|
|
1439
|
+
assert: (rows) => {
|
|
1440
|
+
const r = rows?.[0];
|
|
1441
|
+
if (!r || Number(r.over_submits) < 10000 || Number(r.sweet_submits) < 8000) {
|
|
1442
|
+
return { verdict: "WEAK", detail: `submit volume too small: over=${r?.over_submits ?? 0} sweet=${r?.sweet_submits ?? 0}` };
|
|
1443
|
+
}
|
|
1444
|
+
const ratio = Number(r.over_aps) / Number(r.sweet_aps);
|
|
1445
|
+
const detail = `over/sweet approvals-per-submit=${ratio.toFixed(3)} (mechanism 0.6 x selection ~0.8-1.0; submits=${r.over_submits}/${r.sweet_submits})`;
|
|
1446
|
+
if (ratio >= 0.45 && ratio <= 0.66) return { verdict: "NAILED", detail };
|
|
1447
|
+
if (ratio >= 0.38 && ratio <= 0.75) return { verdict: "STRONG", detail };
|
|
1448
|
+
if (ratio < 0.85) return { verdict: "WEAK", detail };
|
|
1449
|
+
return { verdict: "INVERSE", detail };
|
|
1450
|
+
},
|
|
1451
|
+
},
|
|
1452
|
+
],
|
|
1453
|
+
},
|
|
1454
|
+
{
|
|
1455
|
+
id: "H5-ttc-account-type",
|
|
1456
|
+
hook: "H5",
|
|
1457
|
+
archetype: "funnel-ttc-by-segment",
|
|
1458
|
+
narrative:
|
|
1459
|
+
"Every 'application approved' is pinned to firstStart + {36,48,63}h by account_type " +
|
|
1460
|
+
"(djb2 hash of user_id) + jitter in [0,4h) — support is EXACTLY [target, target+4h). " +
|
|
1461
|
+
"Medians land at target + ~1-2h; ratios biz/indiv ≈ (36+j)/(48+j) ≈ 0.75, fam/indiv ≈ " +
|
|
1462
|
+
"(63+j)/(48+j) ≈ 1.30. account_type is only visible on born users' 'account created' " +
|
|
1463
|
+
"events, so the breakdown covers born users only (pre-existing users are engineered " +
|
|
1464
|
+
"identically but unlabeled).",
|
|
1465
|
+
assertions: [
|
|
1466
|
+
{
|
|
1467
|
+
breakdown: {
|
|
1468
|
+
type: "duckdb",
|
|
1469
|
+
sql: `WITH ${ID_CTE},
|
|
1470
|
+
acct AS (SELECT uid, min(account_type) AS account_type FROM ev WHERE event = 'account created' GROUP BY 1),
|
|
1471
|
+
fs AS (SELECT uid, min(t) AS first_started FROM ev WHERE event = 'application started' GROUP BY 1),
|
|
1472
|
+
gaps AS (SELECT a.account_type, e.uid, date_diff('second', f.first_started, e.t) / 3600.0 AS gap_h
|
|
1473
|
+
FROM ev e JOIN fs f ON e.uid = f.uid JOIN acct a ON e.uid = a.uid
|
|
1474
|
+
WHERE e.event = 'application approved')
|
|
1475
|
+
SELECT account_type, count(*)::BIGINT AS n, count(DISTINCT uid)::BIGINT AS users, median(gap_h) AS med
|
|
1476
|
+
FROM gaps GROUP BY 1`,
|
|
1477
|
+
},
|
|
1478
|
+
assert: (rows) => {
|
|
1479
|
+
const by = cellsOf(rows, "account_type");
|
|
1480
|
+
const b = by.business, i = by.individual, f = by.family;
|
|
1481
|
+
if (!b || !i || !f || Number(b.users) < 40 || Number(i.users) < 40 || Number(f.users) < 40) {
|
|
1482
|
+
return { verdict: "WEAK", detail: `account_type cohorts too small: biz=${b?.users ?? 0} indiv=${i?.users ?? 0} fam=${f?.users ?? 0}` };
|
|
1483
|
+
}
|
|
1484
|
+
const mb = Number(b.med), mi = Number(i.med), mf = Number(f.med);
|
|
1485
|
+
const detail = `median started→approved gap: biz=${mb.toFixed(1)}h indiv=${mi.toFixed(1)}h fam=${mf.toFixed(1)}h (targets ${TTC_BUSINESS_HOURS}/${TTC_INDIVIDUAL_HOURS}/${TTC_FAMILY_HOURS}+jitter; users=${b.users}/${i.users}/${f.users})`;
|
|
1486
|
+
const inBand = (m, t) => m >= t && m <= t + 4;
|
|
1487
|
+
if (inBand(mb, TTC_BUSINESS_HOURS) && inBand(mi, TTC_INDIVIDUAL_HOURS) && inBand(mf, TTC_FAMILY_HOURS)) {
|
|
1488
|
+
return { verdict: "NAILED", detail };
|
|
1489
|
+
}
|
|
1490
|
+
const near = (m, t) => m >= t - 1 && m <= t + 6;
|
|
1491
|
+
if (near(mb, TTC_BUSINESS_HOURS) && near(mi, TTC_INDIVIDUAL_HOURS) && near(mf, TTC_FAMILY_HOURS)) {
|
|
1492
|
+
return { verdict: "STRONG", detail };
|
|
1493
|
+
}
|
|
1494
|
+
if (mb < mi && mi < mf) return { verdict: "WEAK", detail };
|
|
1495
|
+
return { verdict: "INVERSE", detail };
|
|
1496
|
+
},
|
|
1497
|
+
},
|
|
1498
|
+
{
|
|
1499
|
+
breakdown: {
|
|
1500
|
+
type: "duckdb",
|
|
1501
|
+
sql: `WITH ${ID_CTE},
|
|
1502
|
+
acct AS (SELECT uid, min(account_type) AS account_type FROM ev WHERE event = 'account created' GROUP BY 1),
|
|
1503
|
+
fs AS (SELECT uid, min(t) AS first_started FROM ev WHERE event = 'application started' GROUP BY 1),
|
|
1504
|
+
gaps AS (SELECT a.account_type, e.uid, date_diff('second', f.first_started, e.t) / 3600.0 AS gap_h
|
|
1505
|
+
FROM ev e JOIN fs f ON e.uid = f.uid JOIN acct a ON e.uid = a.uid
|
|
1506
|
+
WHERE e.event = 'application approved')
|
|
1507
|
+
SELECT account_type, count(DISTINCT uid)::BIGINT AS users, median(gap_h) AS med
|
|
1508
|
+
FROM gaps GROUP BY 1`,
|
|
1509
|
+
},
|
|
1510
|
+
assert: (rows) => {
|
|
1511
|
+
const by = cellsOf(rows, "account_type");
|
|
1512
|
+
const b = by.business, i = by.individual, f = by.family;
|
|
1513
|
+
if (!b || !i || !f || Number(b.users) < 40 || Number(i.users) < 40 || Number(f.users) < 40) {
|
|
1514
|
+
return { verdict: "WEAK", detail: `account_type cohorts too small: biz=${b?.users ?? 0} indiv=${i?.users ?? 0} fam=${f?.users ?? 0}` };
|
|
1515
|
+
}
|
|
1516
|
+
const rb = Number(b.med) / Number(i.med), rf = Number(f.med) / Number(i.med);
|
|
1517
|
+
const detail = `TTC ratios biz/indiv=${rb.toFixed(3)} fam/indiv=${rf.toFixed(3)} (mechanism ~0.75 / ~1.30)`;
|
|
1518
|
+
if (rb >= 0.72 && rb <= 0.78 && rf >= 1.26 && rf <= 1.35) return { verdict: "NAILED", detail };
|
|
1519
|
+
if (rb >= 0.68 && rb <= 0.82 && rf >= 1.2 && rf <= 1.42) return { verdict: "STRONG", detail };
|
|
1520
|
+
if (rb < 1 && rf > 1) return { verdict: "WEAK", detail };
|
|
1521
|
+
return { verdict: "INVERSE", detail };
|
|
1522
|
+
},
|
|
1523
|
+
},
|
|
1524
|
+
],
|
|
1525
|
+
},
|
|
1526
|
+
{
|
|
1527
|
+
id: "H6-claims-experiment",
|
|
1528
|
+
hook: "H6",
|
|
1529
|
+
archetype: "experiment-lift",
|
|
1530
|
+
narrative:
|
|
1531
|
+
"Engine-native A/B on the Claims Process funnel (no hook code): variant = " +
|
|
1532
|
+
"quickHash(userId:expName) % 2 (weights default 1 → 50/50), $experiment_started fires " +
|
|
1533
|
+
"for every in-window instance and is gated on firstEventTime >= startUnix (end − 35d), " +
|
|
1534
|
+
"so ZERO exposures precede the boundary. Simplified Claims: conversionRate " +
|
|
1535
|
+
"min(100, round(50x1.3)) = 65 vs Control 50 → completion ratio mechanism 1.3, read via " +
|
|
1536
|
+
"ordered 3-step completion within 49h of each post-boundary 'claim filed' (49h ≈ 2x the " +
|
|
1537
|
+
"24h funnel TTC, covering the 0.8x-compressed Simplified support with less censoring " +
|
|
1538
|
+
"than a 25h read).",
|
|
1539
|
+
assertions: [
|
|
1540
|
+
{
|
|
1541
|
+
breakdown: {
|
|
1542
|
+
type: "duckdb",
|
|
1543
|
+
sql: `WITH ${ID_CTE}
|
|
1544
|
+
SELECT "Variant name" AS variant, count(*)::BIGINT AS exposures, count(DISTINCT uid)::BIGINT AS users,
|
|
1545
|
+
count(*) FILTER (WHERE t < TIMESTAMP '${EXP_START_TS}')::BIGINT AS early_n
|
|
1546
|
+
FROM ev WHERE event = '$experiment_started' GROUP BY 1`,
|
|
1547
|
+
},
|
|
1548
|
+
assert: (rows) => {
|
|
1549
|
+
const by = cellsOf(rows, "variant");
|
|
1550
|
+
const c = by["Control"], s = by["Simplified Claims"];
|
|
1551
|
+
if (!c || !s || Number(c.users) + Number(s.users) < 300) {
|
|
1552
|
+
return { verdict: "WEAK", detail: `exposed users too few: control=${c?.users ?? 0} simplified=${s?.users ?? 0}` };
|
|
1553
|
+
}
|
|
1554
|
+
const early = Number(c.early_n) + Number(s.early_n);
|
|
1555
|
+
const share = Number(c.users) / (Number(c.users) + Number(s.users));
|
|
1556
|
+
const minority = Math.min(share, 1 - share);
|
|
1557
|
+
const detail = `variant users control=${c.users} simplified=${s.users} (minority share=${minority.toFixed(3)}); pre-start exposures=${early} (expect exactly 0)`;
|
|
1558
|
+
if (early === 0 && minority >= 0.44) return { verdict: "NAILED", detail };
|
|
1559
|
+
if (early === 0 && minority >= 0.4) return { verdict: "STRONG", detail };
|
|
1560
|
+
if (early === 0) return { verdict: "WEAK", detail };
|
|
1561
|
+
return { verdict: "INVERSE", detail };
|
|
1562
|
+
},
|
|
1563
|
+
},
|
|
1564
|
+
{
|
|
1565
|
+
breakdown: {
|
|
1566
|
+
type: "duckdb",
|
|
1567
|
+
sql: `WITH ${ID_CTE},
|
|
1568
|
+
expusers AS (SELECT uid, min("Variant name") AS variant FROM ev WHERE event = '$experiment_started'
|
|
1569
|
+
GROUP BY 1 HAVING count(DISTINCT "Variant name") = 1),
|
|
1570
|
+
claimwin AS (SELECT c.uid,
|
|
1571
|
+
(EXISTS (SELECT 1 FROM ev s WHERE s.uid = c.uid AND s.event = 'claim status checked'
|
|
1572
|
+
AND s.t > c.t AND s.t <= c.t + INTERVAL 49 HOUR
|
|
1573
|
+
AND EXISTS (SELECT 1 FROM ev k WHERE k.uid = c.uid AND k.event = 'support ticket created'
|
|
1574
|
+
AND k.t > s.t AND k.t <= c.t + INTERVAL 49 HOUR))) AS completed
|
|
1575
|
+
FROM ev c WHERE c.event = 'claim filed' AND c.t >= TIMESTAMP '${EXP_START_TS}')
|
|
1576
|
+
SELECT x.variant, count(*)::BIGINT AS instances, avg(w.completed::INT) AS completion
|
|
1577
|
+
FROM claimwin w JOIN expusers x ON w.uid = x.uid GROUP BY 1`,
|
|
1578
|
+
},
|
|
1579
|
+
assert: (rows) => {
|
|
1580
|
+
const by = cellsOf(rows, "variant");
|
|
1581
|
+
const c = by["Control"], s = by["Simplified Claims"];
|
|
1582
|
+
if (!c || !s || Number(c.instances) < 700 || Number(s.instances) < 700) {
|
|
1583
|
+
return { verdict: "WEAK", detail: `claim instances too few: control=${c?.instances ?? 0} simplified=${s?.instances ?? 0}` };
|
|
1584
|
+
}
|
|
1585
|
+
const ratio = Number(s.completion) / Number(c.completion);
|
|
1586
|
+
const detail = `49h claims completion simplified/control=${ratio.toFixed(3)} (mechanism 1.3 = 65/50; instances=${s.instances}/${c.instances})`;
|
|
1587
|
+
if (ratio >= 1.15 && ratio <= 1.55) return { verdict: "NAILED", detail };
|
|
1588
|
+
if (ratio >= 1.05 && ratio <= 1.75) return { verdict: "STRONG", detail };
|
|
1589
|
+
if (ratio > 1) return { verdict: "WEAK", detail };
|
|
1590
|
+
return { verdict: "INVERSE", detail };
|
|
1591
|
+
},
|
|
1592
|
+
},
|
|
1593
|
+
],
|
|
1594
|
+
},
|
|
1595
|
+
{
|
|
1596
|
+
id: "H7-risk-approval",
|
|
1597
|
+
hook: "H7",
|
|
1598
|
+
archetype: "funnel-conversion-by-segment",
|
|
1599
|
+
narrative:
|
|
1600
|
+
"funnel-pre multiplies the Application Approval funnel's conversionRate (base 70): " +
|
|
1601
|
+
"low = min(95, round(70x1.8)) = 95, high = round(70x0.3) = 21, medium 70. " +
|
|
1602
|
+
"Non-converting instances still take a uniform 1..(steps-1) partial walk " +
|
|
1603
|
+
"(determineConversion, lib/generators/funnels.js:527-534), so step 2 fires with " +
|
|
1604
|
+
"p = c + (1-c)/2 → 0.975 / 0.85 / 0.605, compressing observable approvals-per-submit " +
|
|
1605
|
+
"ratios to low/med ≈ 1.147 and high/med ≈ 0.712 (NOT the naive 1.8/0.3). Submit " +
|
|
1606
|
+
"denominators include Application Completion conversions for all groups equally, " +
|
|
1607
|
+
"preserving the ratios.",
|
|
1608
|
+
assertions: [
|
|
1609
|
+
{
|
|
1610
|
+
breakdown: {
|
|
1611
|
+
type: "duckdb",
|
|
1612
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1613
|
+
SELECT u.risk_profile, count(*)::BIGINT AS users,
|
|
1614
|
+
sum(p.approvals)::DOUBLE / nullif(sum(p.submits), 0) AS aps
|
|
1615
|
+
FROM puu p JOIN us u ON p.uid = u.duid GROUP BY 1`,
|
|
1616
|
+
},
|
|
1617
|
+
assert: (rows) => {
|
|
1618
|
+
const by = cellsOf(rows, "risk_profile");
|
|
1619
|
+
const lo = by.low, md = by.medium;
|
|
1620
|
+
if (!lo || !md || Number(lo.users) < 400 || Number(md.users) < 400) {
|
|
1621
|
+
return { verdict: "WEAK", detail: `risk cohorts too small: low=${lo?.users ?? 0} med=${md?.users ?? 0}` };
|
|
1622
|
+
}
|
|
1623
|
+
const ratio = Number(lo.aps) / Number(md.aps);
|
|
1624
|
+
const detail = `low/med approvals-per-submit=${ratio.toFixed(3)} (mechanism 0.975/0.85=1.147; users=${lo.users}/${md.users})`;
|
|
1625
|
+
if (ratio >= 1.1 && ratio <= 1.25) return { verdict: "NAILED", detail };
|
|
1626
|
+
if (ratio >= 1.04 && ratio <= 1.35) return { verdict: "STRONG", detail };
|
|
1627
|
+
if (ratio > 1) return { verdict: "WEAK", detail };
|
|
1628
|
+
return { verdict: "INVERSE", detail };
|
|
1629
|
+
},
|
|
1630
|
+
},
|
|
1631
|
+
{
|
|
1632
|
+
breakdown: {
|
|
1633
|
+
type: "duckdb",
|
|
1634
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1635
|
+
SELECT u.risk_profile, count(*)::BIGINT AS users,
|
|
1636
|
+
sum(p.approvals)::DOUBLE / nullif(sum(p.submits), 0) AS aps
|
|
1637
|
+
FROM puu p JOIN us u ON p.uid = u.duid GROUP BY 1`,
|
|
1638
|
+
},
|
|
1639
|
+
assert: (rows) => {
|
|
1640
|
+
const by = cellsOf(rows, "risk_profile");
|
|
1641
|
+
const hi = by.high, md = by.medium;
|
|
1642
|
+
if (!hi || !md || Number(hi.users) < 400 || Number(md.users) < 400) {
|
|
1643
|
+
return { verdict: "WEAK", detail: `risk cohorts too small: high=${hi?.users ?? 0} med=${md?.users ?? 0}` };
|
|
1644
|
+
}
|
|
1645
|
+
const ratio = Number(hi.aps) / Number(md.aps);
|
|
1646
|
+
const detail = `high/med approvals-per-submit=${ratio.toFixed(3)} (mechanism 0.605/0.85=0.712; users=${hi.users}/${md.users})`;
|
|
1647
|
+
if (ratio >= 0.65 && ratio <= 0.79) return { verdict: "NAILED", detail };
|
|
1648
|
+
if (ratio >= 0.57 && ratio <= 0.87) return { verdict: "STRONG", detail };
|
|
1649
|
+
if (ratio < 0.95) return { verdict: "WEAK", detail };
|
|
1650
|
+
return { verdict: "INVERSE", detail };
|
|
1651
|
+
},
|
|
1652
|
+
},
|
|
1653
|
+
],
|
|
1654
|
+
},
|
|
1655
|
+
{
|
|
1656
|
+
id: "H8-doc-retention",
|
|
1657
|
+
hook: "H8",
|
|
1658
|
+
archetype: "retention-divergence",
|
|
1659
|
+
narrative:
|
|
1660
|
+
"Born users with <3 'document uploaded' in their first 14 days lose 75% of events after " +
|
|
1661
|
+
"day 30 (post-day-30 keep-rate 0.25). Read as difference-in-differences: " +
|
|
1662
|
+
"(non-uploader post/pre event ratio) ÷ (uploader post/pre ratio) — the uploader arm " +
|
|
1663
|
+
"cancels organic trajectory. Mechanism 0.25 x organic-DiD ~0.8-1.5 → band [0.20, 0.38]. " +
|
|
1664
|
+
"Population: born before 2026-03-02 so a post-window exists. Cohorts are small even at " +
|
|
1665
|
+
"15K (~240 non-uploaders / ~45 uploaders): guards sized for full fidelity; reduced-scale " +
|
|
1666
|
+
"runs hit scale-cap WEAK.",
|
|
1667
|
+
assertions: [
|
|
1668
|
+
{
|
|
1669
|
+
breakdown: {
|
|
1670
|
+
type: "duckdb",
|
|
1671
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE},
|
|
1672
|
+
h8pop AS (SELECT p.*,
|
|
1673
|
+
(SELECT count(*) FROM ev e WHERE e.uid = p.uid AND e.t <= p.first_t + INTERVAL '${DOC_RETENTION_CUTOFF_DAYS} days') AS pre_n,
|
|
1674
|
+
(SELECT count(*) FROM ev e WHERE e.uid = p.uid AND e.t > p.first_t + INTERVAL '${DOC_RETENTION_CUTOFF_DAYS} days') AS post_n
|
|
1675
|
+
FROM puu p WHERE p.born AND p.first_t < TIMESTAMP '2026-03-02')
|
|
1676
|
+
SELECT
|
|
1677
|
+
(SELECT sum(post_n)::DOUBLE / nullif(sum(pre_n), 0) FROM h8pop WHERE NOT uploader) AS nonup_pp,
|
|
1678
|
+
(SELECT sum(post_n)::DOUBLE / nullif(sum(pre_n), 0) FROM h8pop WHERE uploader) AS up_pp,
|
|
1679
|
+
(SELECT count(*) FROM h8pop WHERE NOT uploader)::BIGINT AS nonup_users,
|
|
1680
|
+
(SELECT count(*) FROM h8pop WHERE uploader)::BIGINT AS up_users`,
|
|
1681
|
+
},
|
|
1682
|
+
assert: (rows) => {
|
|
1683
|
+
const r = rows?.[0];
|
|
1684
|
+
if (!r || Number(r.nonup_users) < 150 || Number(r.up_users) < 25) {
|
|
1685
|
+
return { verdict: "WEAK", detail: `retention cohorts too small: nonup=${r?.nonup_users ?? 0} up=${r?.up_users ?? 0}` };
|
|
1686
|
+
}
|
|
1687
|
+
const did = Number(r.nonup_pp) / Number(r.up_pp);
|
|
1688
|
+
const detail = `churn DiD=${did.toFixed(3)} (nonup post/pre=${Number(r.nonup_pp).toFixed(3)} ÷ up=${Number(r.up_pp).toFixed(3)}; mechanism 0.25; n=${r.nonup_users}/${r.up_users})`;
|
|
1689
|
+
if (did >= 0.2 && did <= 0.38) return { verdict: "NAILED", detail };
|
|
1690
|
+
if (did >= 0.14 && did <= 0.5) return { verdict: "STRONG", detail };
|
|
1691
|
+
if (did < 0.7) return { verdict: "WEAK", detail };
|
|
1692
|
+
return { verdict: "INVERSE", detail };
|
|
1693
|
+
},
|
|
1694
|
+
},
|
|
1695
|
+
{
|
|
1696
|
+
breakdown: {
|
|
1697
|
+
type: "duckdb",
|
|
1698
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE},
|
|
1699
|
+
h8pop AS (SELECT p.*,
|
|
1700
|
+
(SELECT count(*) FROM ev e WHERE e.uid = p.uid AND e.t <= p.first_t + INTERVAL '${DOC_RETENTION_CUTOFF_DAYS} days') AS pre_n,
|
|
1701
|
+
(SELECT count(*) FROM ev e WHERE e.uid = p.uid AND e.t > p.first_t + INTERVAL '${DOC_RETENTION_CUTOFF_DAYS} days') AS post_n
|
|
1702
|
+
FROM puu p WHERE p.born AND p.first_t < TIMESTAMP '2026-03-02')
|
|
1703
|
+
SELECT
|
|
1704
|
+
(SELECT sum(post_n)::DOUBLE / nullif(sum(pre_n), 0) FROM h8pop WHERE NOT uploader) AS nonup_pp,
|
|
1705
|
+
(SELECT sum(post_n)::DOUBLE / nullif(sum(pre_n), 0) FROM h8pop WHERE uploader) AS up_pp,
|
|
1706
|
+
(SELECT count(*) FROM h8pop WHERE NOT uploader)::BIGINT AS nonup_users,
|
|
1707
|
+
(SELECT count(*) FROM h8pop WHERE uploader)::BIGINT AS up_users`,
|
|
1708
|
+
},
|
|
1709
|
+
assert: (rows) => {
|
|
1710
|
+
const r = rows?.[0];
|
|
1711
|
+
if (!r || Number(r.nonup_users) < 150 || Number(r.up_users) < 25) {
|
|
1712
|
+
return { verdict: "WEAK", detail: `retention cohorts too small: nonup=${r?.nonup_users ?? 0} up=${r?.up_users ?? 0}` };
|
|
1713
|
+
}
|
|
1714
|
+
const sep = Number(r.up_pp) / Number(r.nonup_pp);
|
|
1715
|
+
const detail = `uploader/non-uploader post-pre separation=${sep.toFixed(2)}x (retention-curve split; n=${r.up_users}/${r.nonup_users})`;
|
|
1716
|
+
if (sep >= 2.5) return { verdict: "NAILED", detail };
|
|
1717
|
+
if (sep >= 1.8) return { verdict: "STRONG", detail };
|
|
1718
|
+
if (sep > 1.2) return { verdict: "WEAK", detail };
|
|
1719
|
+
return { verdict: "INVERSE", detail };
|
|
1720
|
+
},
|
|
1721
|
+
},
|
|
1722
|
+
],
|
|
1723
|
+
},
|
|
1724
|
+
{
|
|
1725
|
+
id: "H9-renewal-spike",
|
|
1726
|
+
hook: "H9",
|
|
1727
|
+
archetype: "temporal-inflection",
|
|
1728
|
+
narrative:
|
|
1729
|
+
"Days 85-94 (window (2026-03-27, 2026-04-06) strict): each 'renewal completed' gets " +
|
|
1730
|
+
"+2 clones (3x), each 'coverage reviewed' +1 (2x). Baseline is LOCAL (days 75-84 ∪ " +
|
|
1731
|
+
"96-105) to cancel dataset-level growth; day 95 is excluded because clone jitter " +
|
|
1732
|
+
"(+5-120min) bleeds boundary events into Apr 6 early morning. Coverage reads slightly " +
|
|
1733
|
+
"under 2x: the window is weekend-heavy (Fri/Sat/Sun = 6 of 10 days vs 3 of 7 baseline), " +
|
|
1734
|
+
"depressing the organic in-window rate that the doubling multiplies.",
|
|
1735
|
+
assertions: [
|
|
1736
|
+
{
|
|
1737
|
+
breakdown: {
|
|
1738
|
+
type: "duckdb",
|
|
1739
|
+
sql: `WITH ${ID_CTE}
|
|
1740
|
+
SELECT count(*) FILTER (WHERE t > TIMESTAMP '2026-03-27' AND t < TIMESTAMP '2026-04-06')::BIGINT AS spike_n,
|
|
1741
|
+
count(*) FILTER (WHERE (day_idx BETWEEN 75 AND 84) OR (day_idx BETWEEN 96 AND 105))::BIGINT AS local_n
|
|
1742
|
+
FROM ev WHERE event = 'renewal completed'`,
|
|
1743
|
+
},
|
|
1744
|
+
assert: (rows) => {
|
|
1745
|
+
const r = rows?.[0];
|
|
1746
|
+
if (!r || Number(r.spike_n) < 2000 || Number(r.local_n) < 1500) {
|
|
1747
|
+
return { verdict: "WEAK", detail: `renewal volume too small: spike=${r?.spike_n ?? 0} local=${r?.local_n ?? 0}` };
|
|
1748
|
+
}
|
|
1749
|
+
const ratio = (Number(r.spike_n) / 10.0) / (Number(r.local_n) / 20.0);
|
|
1750
|
+
const detail = `renewal spike/local=${ratio.toFixed(3)} (mechanism ${1 + RENEWAL_CLONE_COUNT}x; n=${r.spike_n}/${r.local_n})`;
|
|
1751
|
+
// Fix-round Q5 (S1): NAILED band is the knob ±10% — 3x → [2.7, 3.3].
|
|
1752
|
+
if (ratio >= 2.7 && ratio <= 3.3) return { verdict: "NAILED", detail };
|
|
1753
|
+
if (ratio >= 2.3 && ratio <= 3.9) return { verdict: "STRONG", detail };
|
|
1754
|
+
if (ratio >= 1.5) return { verdict: "WEAK", detail };
|
|
1755
|
+
return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
|
|
1756
|
+
},
|
|
1757
|
+
},
|
|
1758
|
+
{
|
|
1759
|
+
breakdown: {
|
|
1760
|
+
type: "duckdb",
|
|
1761
|
+
sql: `WITH ${ID_CTE}
|
|
1762
|
+
SELECT count(*) FILTER (WHERE t > TIMESTAMP '2026-03-27' AND t < TIMESTAMP '2026-04-06')::BIGINT AS spike_n,
|
|
1763
|
+
count(*) FILTER (WHERE (day_idx BETWEEN 75 AND 84) OR (day_idx BETWEEN 96 AND 105))::BIGINT AS local_n
|
|
1764
|
+
FROM ev WHERE event = 'coverage reviewed'`,
|
|
1765
|
+
},
|
|
1766
|
+
assert: (rows) => {
|
|
1767
|
+
const r = rows?.[0];
|
|
1768
|
+
if (!r || Number(r.spike_n) < 2000 || Number(r.local_n) < 2000) {
|
|
1769
|
+
return { verdict: "WEAK", detail: `coverage volume too small: spike=${r?.spike_n ?? 0} local=${r?.local_n ?? 0}` };
|
|
1770
|
+
}
|
|
1771
|
+
const ratio = (Number(r.spike_n) / 10.0) / (Number(r.local_n) / 20.0);
|
|
1772
|
+
const detail = `coverage spike/local=${ratio.toFixed(3)} (mechanism ${1 + COVERAGE_CLONE_COUNT}x, weekend-heavy window drags low; n=${r.spike_n}/${r.local_n})`;
|
|
1773
|
+
// Fix-round Q5 (S1): NAILED band re-derived from the knob (2x ±10% =
|
|
1774
|
+
// [1.8, 2.2]), replacing the measurement-wrapped [1.6, 2.1] that let
|
|
1775
|
+
// 20%-below-knob read NAILED by construction. The weekend-heavy window
|
|
1776
|
+
// genuinely drags the realized ratio below the knob (mechanism in the
|
|
1777
|
+
// narrative), so at 2K fidelity this assertion is EXPECTED to land
|
|
1778
|
+
// STRONG (~1.78) — the honest verdict for a knob whose realized effect
|
|
1779
|
+
// is confounded by DOW weighting.
|
|
1780
|
+
if (ratio >= 1.8 && ratio <= 2.2) return { verdict: "NAILED", detail };
|
|
1781
|
+
if (ratio >= 1.4 && ratio <= 2.4) return { verdict: "STRONG", detail };
|
|
1782
|
+
if (ratio >= 1.2) return { verdict: "WEAK", detail };
|
|
1783
|
+
return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
|
|
1784
|
+
},
|
|
1785
|
+
},
|
|
1786
|
+
],
|
|
1787
|
+
},
|
|
1788
|
+
{
|
|
1789
|
+
id: "H10-claim-premium",
|
|
1790
|
+
hook: "H10",
|
|
1791
|
+
archetype: "cohort-prop-scale",
|
|
1792
|
+
narrative:
|
|
1793
|
+
"First 'payment made' after each 'claim filed' gets premium_amount x2 (one-shot Map " +
|
|
1794
|
+
"consumption). Organic premium_amount caps at 600, so payments > 600 are claim-inflated " +
|
|
1795
|
+
"BY CONSTRUCTION: ~zero among H8-untouched non-claimants (the event hook fires in " +
|
|
1796
|
+
"generation order, so a claim generated past dataset end can double the next-generated " +
|
|
1797
|
+
"payment and then be dropped by the future-time guard — ~1 orphan in 15K users; band " +
|
|
1798
|
+
"0=NAILED, ≤2=STRONG sized for exactly this). Cohort-average ratio " +
|
|
1799
|
+
"reads ~1.5x, not 2.0x: claims and payments interleave (~7.5 claims / ~9.4 payments per " +
|
|
1800
|
+
"claimant), so roughly half of claimant payments get doubled; the 2.0x lives on the " +
|
|
1801
|
+
"doubled payments themselves.",
|
|
1802
|
+
assertions: [
|
|
1803
|
+
{
|
|
1804
|
+
breakdown: {
|
|
1805
|
+
type: "duckdb",
|
|
1806
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1807
|
+
SELECT
|
|
1808
|
+
(SELECT count(*) FROM ev e JOIN puu p ON e.uid = p.uid
|
|
1809
|
+
WHERE e.event = 'payment made' AND e.premium_amount > 600 AND p.claims = 0
|
|
1810
|
+
AND ((NOT p.born) OR p.uploader))::BIGINT AS gt600_untouched_nonclaim,
|
|
1811
|
+
(SELECT count(*) FROM ev e JOIN puu p ON e.uid = p.uid
|
|
1812
|
+
WHERE e.event = 'payment made' AND e.premium_amount > 600 AND p.claims > 0)::BIGINT AS gt600_claimants`,
|
|
1813
|
+
},
|
|
1814
|
+
assert: (rows) => {
|
|
1815
|
+
const r = rows?.[0];
|
|
1816
|
+
if (!r || Number(r.gt600_claimants) < 2000) {
|
|
1817
|
+
return { verdict: "WEAK", detail: `doubled-payment volume too small: claimant gt600=${r?.gt600_claimants ?? 0}` };
|
|
1818
|
+
}
|
|
1819
|
+
const orphans = Number(r.gt600_untouched_nonclaim);
|
|
1820
|
+
const detail = `payments>600: untouched non-claimants=${orphans} (expect exactly 0), claimants=${r.gt600_claimants}`;
|
|
1821
|
+
if (orphans === 0) return { verdict: "NAILED", detail };
|
|
1822
|
+
if (orphans <= 2) return { verdict: "STRONG", detail };
|
|
1823
|
+
if (orphans <= Number(r.gt600_claimants) * 0.01) return { verdict: "WEAK", detail };
|
|
1824
|
+
return { verdict: "INVERSE", detail };
|
|
1825
|
+
},
|
|
1826
|
+
},
|
|
1827
|
+
{
|
|
1828
|
+
breakdown: {
|
|
1829
|
+
type: "duckdb",
|
|
1830
|
+
sql: `WITH ${ID_CTE}, ${PU_CTE}
|
|
1831
|
+
SELECT
|
|
1832
|
+
(SELECT avg(e.premium_amount) FROM ev e JOIN puu p ON e.uid = p.uid WHERE e.event = 'payment made' AND p.claims > 0) AS claimant_avg,
|
|
1833
|
+
(SELECT avg(e.premium_amount) FROM ev e JOIN puu p ON e.uid = p.uid WHERE e.event = 'payment made' AND p.claims = 0) AS nonclaim_avg,
|
|
1834
|
+
(SELECT count(*) FROM ev e JOIN puu p ON e.uid = p.uid WHERE e.event = 'payment made' AND p.claims > 0)::BIGINT AS claimant_pay_n,
|
|
1835
|
+
(SELECT count(*) FROM ev e JOIN puu p ON e.uid = p.uid WHERE e.event = 'payment made' AND p.claims = 0)::BIGINT AS nonclaim_pay_n`,
|
|
1836
|
+
},
|
|
1837
|
+
assert: (rows) => {
|
|
1838
|
+
const r = rows?.[0];
|
|
1839
|
+
if (!r || Number(r.claimant_pay_n) < 10000 || Number(r.nonclaim_pay_n) < 200) {
|
|
1840
|
+
return { verdict: "WEAK", detail: `payment cohorts too small: claimant=${r?.claimant_pay_n ?? 0} nonclaim=${r?.nonclaim_pay_n ?? 0}` };
|
|
1841
|
+
}
|
|
1842
|
+
const ratio = Number(r.claimant_avg) / Number(r.nonclaim_avg);
|
|
1843
|
+
const detail = `claimant/non-claimant avg premium_amount=${ratio.toFixed(3)} (knob ${POST_CLAIM_PREMIUM_MULT}x per doubled payment, ~half doubled → ~1.5x cohort; n=${r.claimant_pay_n}/${r.nonclaim_pay_n})`;
|
|
1844
|
+
if (ratio >= 1.35 && ratio <= 1.6) return { verdict: "NAILED", detail };
|
|
1845
|
+
if (ratio >= 1.25 && ratio <= 1.75) return { verdict: "STRONG", detail };
|
|
1846
|
+
if (ratio > 1.1) return { verdict: "WEAK", detail };
|
|
1847
|
+
return { verdict: "INVERSE", detail };
|
|
1848
|
+
},
|
|
1849
|
+
},
|
|
1850
|
+
],
|
|
1851
|
+
},
|
|
1852
|
+
];
|