@ak--47/dungeon-master 1.5.4 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +21 -12
  2. package/.claude/skills/create-dungeon/SKILL.md +65 -5
  3. package/.claude/skills/create-project/SKILL.md +1 -1
  4. package/.claude/skills/create-project/provision.mjs +57 -10
  5. package/.claude/skills/verify-dungeon/SKILL.md +56 -25
  6. package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
  7. package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
  8. package/.claude/skills/write-hooks/SKILL.md +156 -18
  9. package/CHANGELOG.md +183 -0
  10. package/HOOKS.md +566 -112
  11. package/README.md +28 -9
  12. package/docs/guides/1.3.0-upgrade-guide.md +262 -0
  13. package/docs/guides/1.3.1-upgrade-guide.md +328 -0
  14. package/docs/guides/1.4.0-upgrade-guide.md +130 -0
  15. package/docs/guides/1.4.1-upgrade-guide.md +125 -0
  16. package/docs/guides/1.4.3-upgrade-guide.md +200 -0
  17. package/docs/guides/1.4.4-upgrade-guide.md +95 -0
  18. package/docs/guides/1.5.0-upgrade-guide.md +426 -0
  19. package/docs/guides/1.5.1-upgrade-guide.md +178 -0
  20. package/docs/guides/1.5.3-upgrade-guide.md +117 -0
  21. package/docs/guides/1.6.0-upgrade-guide.md +204 -0
  22. package/dungeons/technical/stories-verify.js +152 -0
  23. package/dungeons/vertical/README.md +154 -0
  24. package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
  25. package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
  26. package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
  27. package/dungeons/vertical/community/community.js +1589 -0
  28. package/dungeons/vertical/community/community.sql +216 -0
  29. package/dungeons/vertical/community/community.verify.mjs +72 -0
  30. package/dungeons/vertical/crypto/crypto.js +1483 -0
  31. package/dungeons/vertical/crypto/crypto.sql +194 -0
  32. package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
  33. package/dungeons/vertical/dating/dating.js +1540 -0
  34. package/dungeons/vertical/dating/dating.sql +288 -0
  35. package/dungeons/vertical/dating/dating.verify.mjs +72 -0
  36. package/dungeons/vertical/devtools/devtools.js +1524 -0
  37. package/dungeons/vertical/devtools/devtools.sql +161 -0
  38. package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
  39. package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
  40. package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
  41. package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
  42. package/dungeons/vertical/education/education.js +1763 -0
  43. package/dungeons/vertical/education/education.sql +243 -0
  44. package/dungeons/vertical/education/education.verify.mjs +75 -0
  45. package/dungeons/vertical/fintech/fintech.js +1410 -0
  46. package/dungeons/vertical/fintech/fintech.sql +175 -0
  47. package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
  48. package/dungeons/vertical/fitness/fitness.js +1495 -0
  49. package/dungeons/vertical/fitness/fitness.sql +206 -0
  50. package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
  51. package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
  52. package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
  53. package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
  54. package/dungeons/vertical/gaming/gaming.js +1741 -0
  55. package/dungeons/vertical/gaming/gaming.sql +249 -0
  56. package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
  57. package/dungeons/vertical/healthcare/healthcare.js +1488 -0
  58. package/dungeons/vertical/healthcare/healthcare.sql +193 -0
  59. package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
  60. package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
  61. package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
  62. package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
  63. package/dungeons/vertical/logistics/logistics.js +1541 -0
  64. package/dungeons/vertical/logistics/logistics.sql +205 -0
  65. package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
  66. package/dungeons/vertical/marketplace/marketplace.js +1438 -0
  67. package/dungeons/vertical/marketplace/marketplace.sql +148 -0
  68. package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
  69. package/dungeons/vertical/{media.js → media/media.js} +557 -58
  70. package/dungeons/vertical/media/media.sql +173 -0
  71. package/dungeons/vertical/media/media.verify.mjs +74 -0
  72. package/dungeons/vertical/real-estate/real-estate.js +1629 -0
  73. package/dungeons/vertical/real-estate/real-estate.sql +167 -0
  74. package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
  75. package/dungeons/vertical/sass/sass.js +1733 -0
  76. package/dungeons/vertical/sass/sass.sql +173 -0
  77. package/dungeons/vertical/sass/sass.verify.mjs +74 -0
  78. package/dungeons/vertical/social/social.js +1456 -0
  79. package/dungeons/vertical/social/social.sql +160 -0
  80. package/dungeons/vertical/social/social.verify.mjs +76 -0
  81. package/dungeons/vertical/streaming/streaming.js +677 -0
  82. package/dungeons/vertical/streaming/streaming.sql +101 -0
  83. package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
  84. package/dungeons/vertical/support-desk/support-desk.js +741 -0
  85. package/dungeons/vertical/support-desk/support-desk.sql +137 -0
  86. package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
  87. package/dungeons/vertical/travel/travel.js +1603 -0
  88. package/dungeons/vertical/travel/travel.sql +236 -0
  89. package/dungeons/vertical/travel/travel.verify.mjs +72 -0
  90. package/index.js +2 -2
  91. package/lib/core/config-validator.js +37 -2
  92. package/lib/core/context.js +4 -2
  93. package/lib/core/dungeon-loader.js +20 -15
  94. package/lib/core/dungeon-to-json.js +22 -4
  95. package/lib/core/storage.js +3 -3
  96. package/lib/generators/funnels.js +36 -9
  97. package/lib/hook-helpers/cohort.js +53 -0
  98. package/lib/hook-helpers/index.js +8 -0
  99. package/lib/hook-helpers/mutate.js +1 -1
  100. package/lib/hook-helpers/shape.js +319 -0
  101. package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
  102. package/lib/hook-patterns/attributed-by-source.js +60 -58
  103. package/lib/hook-patterns/frequency-by-frequency.js +14 -4
  104. package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
  105. package/lib/hook-patterns/index.js +1 -1
  106. package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
  107. package/lib/orchestrators/user-loop.js +41 -0
  108. package/lib/templates/story-spec.schema.json +175 -0
  109. package/lib/utils/json-evaluator.js +1 -1
  110. package/lib/utils/utils.js +5 -4
  111. package/lib/verify/coerce.js +186 -0
  112. package/lib/verify/counting.js +270 -32
  113. package/lib/verify/emulate-breakdown.js +1285 -203
  114. package/lib/verify/first-time.js +89 -0
  115. package/lib/verify/flows.js +679 -0
  116. package/lib/verify/formula.js +259 -0
  117. package/lib/verify/funnel-engine.js +697 -113
  118. package/lib/verify/index.js +32 -4
  119. package/lib/verify/sessionize.js +275 -0
  120. package/lib/verify/story-runner.js +483 -0
  121. package/lib/verify/verify-dungeon.js +59 -37
  122. package/package.json +2 -1
  123. package/scripts/extract-dungeon-schema.mjs +2 -2
  124. package/scripts/run-many.mjs +2 -2
  125. package/scripts/verify-runner.mjs +2 -2
  126. package/scripts/verify-stories.mjs +244 -0
  127. package/types.d.ts +296 -30
  128. package/dungeons/vertical/ai-platform.js +0 -818
  129. package/dungeons/vertical/community.js +0 -966
  130. package/dungeons/vertical/crypto.js +0 -895
  131. package/dungeons/vertical/dating.js +0 -830
  132. package/dungeons/vertical/devtools.js +0 -1050
  133. package/dungeons/vertical/ecommerce.js +0 -811
  134. package/dungeons/vertical/education.js +0 -964
  135. package/dungeons/vertical/fintech.js +0 -946
  136. package/dungeons/vertical/fitness.js +0 -935
  137. package/dungeons/vertical/food-delivery.js +0 -934
  138. package/dungeons/vertical/gaming.js +0 -1120
  139. package/dungeons/vertical/healthcare.js +0 -945
  140. package/dungeons/vertical/insurance-application.js +0 -1039
  141. package/dungeons/vertical/logistics.js +0 -986
  142. package/dungeons/vertical/marketplace.js +0 -1019
  143. package/dungeons/vertical/real-estate.js +0 -872
  144. package/dungeons/vertical/sass.js +0 -995
  145. package/dungeons/vertical/social.js +0 -904
  146. package/dungeons/vertical/travel.js +0 -930
@@ -0,0 +1,1763 @@
1
+ // ── IMPORTS ──
2
+ import dayjs from "dayjs";
3
+ import utc from "dayjs/plugin/utc.js";
4
+ dayjs.extend(utc);
5
+ import "dotenv/config";
6
+ import * as u from "@ak--47/dungeon-master/utils";
7
+ import * as v from "ak-tools";
8
+ /** @typedef {import("../../../types").Dungeon} Config */
9
+
10
+ // ── OVERVIEW ──
11
+ /*
12
+ * NAME: LearnPath
13
+ * APP: Online learning platform modeled after Coursera, Khan Academy,
14
+ * and Udemy. Self-paced and cohort-based courses with quizzes,
15
+ * assignments, certificates, and a social study layer. Two-sided
16
+ * marketplace: ~89% students, ~11% instructors.
17
+ * SCALE: 10,000 users, ~760K events, 121 days (2026-01-01 → 2026-05-01)
18
+ * CORE LOOP: account registered → course enrolled → lecture started/completed → quiz → certificate
19
+ *
20
+ * EVENTS (17):
21
+ * lecture started (18) > lecture completed (14) > practice problem solved (12)
22
+ * > quiz started (10) > resource downloaded (9) > course enrolled (8)
23
+ * > quiz completed (8) > discussion posted (7) > assignment submitted (6)
24
+ * > assignment graded (5) > help requested (4) > study group joined (4)
25
+ * > instructor feedback given (3) > course reviewed (3) > certificate earned (2)
26
+ * > subscription purchased (2) > account registered (1)
27
+ *
28
+ * FUNNELS (7):
29
+ * - Onboarding: account registered → course enrolled → lecture started (75%)
30
+ * - Learning Loop: lecture started → lecture completed → practice problem solved (70%, reentry)
31
+ * - Assessment: quiz started → quiz completed → assignment submitted (55%, reentry)
32
+ * - Course Completion: course enrolled → lecture completed → quiz completed → certificate earned (30%)
33
+ * - Social Learning: discussion posted → study group joined → resource downloaded (50%, AI Study Buddy A/B)
34
+ * - Instructor Interaction: assignment submitted → assignment graded → instructor feedback given (45%)
35
+ * - Support/Monetization: help requested → subscription purchased → course reviewed (35%)
36
+ *
37
+ * USER PROPS: account_type, subscription_status, learning_style, education_level, timezone, courses_created, teaching_experience_years, instructor_rating, learning_goal, study_hours_per_week, Platform
38
+ * SUPER PROPS: Platform
39
+ * SCD PROPS: enrollment_status (enrolled/active/completed/dropped, monthly fuzzy, max 6), course_status (draft/published/archived/deprecated, monthly fixed, max 6, type: course_id)
40
+ * GROUPS: course_id (150 courses), group_id (300 study groups)
41
+ */
42
+
43
+ // ── HOOK STORIES ──
44
+ /*
45
+ * ---------------------------------------------------------------
46
+ * 1. STUDENT VS INSTRUCTOR PROFILES (user)
47
+ * ---------------------------------------------------------------
48
+ *
49
+ * PATTERN: Instructor profiles get teaching attributes
50
+ * (courses_created, teaching_experience_years, instructor_rating).
51
+ * Students get learning attributes (learning_goal,
52
+ * study_hours_per_week). Two-sided marketplace at ~89% students,
53
+ * ~11% instructors. The everything hook also stamps account_type on
54
+ * 'account registered' events from the user's profile (the engine
55
+ * draws event-level pool props independently of profile props), so
56
+ * the event breakdown agrees with the profile breakdown exactly.
57
+ *
58
+ * HOW TO FIND IT IN MIXPANEL:
59
+ *
60
+ * Report 1: Account Mix
61
+ * - Report type: Insights
62
+ * - Event: any event
63
+ * - Measure: Unique users
64
+ * - Breakdown: "account_type"
65
+ * - Expected: ~89% students, ~11% instructors
66
+ *
67
+ * Report 2: Instructor-Driven Feedback
68
+ * - Report type: Insights
69
+ * - Event: "instructor feedback given"
70
+ * - Measure: Total per user (average)
71
+ * - Breakdown: "account_type"
72
+ * - Expected: instructors dominate feedback volume; students rarely emit
73
+ *
74
+ * REAL-WORLD ANALOGUE: Two-sided learning marketplaces have
75
+ * fundamentally different role personas — teachers create supply,
76
+ * learners consume it.
77
+ *
78
+ * ---------------------------------------------------------------
79
+ * 2. DEADLINE CRAMMING (everything)
80
+ * ---------------------------------------------------------------
81
+ *
82
+ * PATTERN: Assignments submitted on Sun/Mon are rushed — 60% are late
83
+ * (raw is_late prop set true at 60% likelihood) vs ~20% baseline. Quiz
84
+ * scores on Sun/Mon drop by 25 points. No flag — discover via Day of Week
85
+ * breakdown.
86
+ *
87
+ * HOW TO FIND IT IN MIXPANEL:
88
+ *
89
+ * Report 1: Late Submission Rate by Day of Week
90
+ * - Report type: Insights
91
+ * - Event: "assignment submitted"
92
+ * - Measure: count where is_late=true / total
93
+ * - Breakdown: Day of Week
94
+ * - Expected: Sun/Mon ~ 60% late vs other days ~ 20%
95
+ *
96
+ * Report 2: Quiz Score by Day of Week
97
+ * - Report type: Insights
98
+ * - Event: "quiz completed"
99
+ * - Measure: Average of "score_percent"
100
+ * - Breakdown: Day of Week
101
+ * - Expected: Sun/Mon ~ 26, other days ~ 50 (-25 knob; the clamp at 0
102
+ * attenuates the observed gap to ~24 pts since organic sub-25 scores
103
+ * can't drop the full amount)
104
+ *
105
+ * REAL-WORLD ANALOGUE: Procrastination clusters submissions at the
106
+ * deadline weekend and hammers performance.
107
+ *
108
+ * ---------------------------------------------------------------
109
+ * 3. NOTES MAGIC NUMBER (everything, in-funnel)
110
+ * ---------------------------------------------------------------
111
+ *
112
+ * PATTERN: Sweet 5-8 lectures with notes_taken=true → +30% quiz
113
+ * score_percent (cap 100) and 40% chance of bonus cloned certificate.
114
+ * Over 9+ → 35% of certificate-earned events drop (over-noted but
115
+ * stuck in study mode). No flag.
116
+ *
117
+ * HOW TO FIND IT IN MIXPANEL:
118
+ *
119
+ * Report 1: Quiz Score by Notes-Taken Bucket
120
+ * - Cohort A: users with 5-8 "lecture completed" where notes_taken=true
121
+ * - Cohort B: users with 0-4
122
+ * - Event: "quiz completed"
123
+ * - Measure: Average of "score_percent"
124
+ * - Expected: A ~ 1.3x B
125
+ *
126
+ * Report 2: Certificates per User on Heavy Note-Takers
127
+ * - Cohort C: users with >= 9 notes-taken lectures
128
+ * - Cohort A: users with 5-8
129
+ * - Event: "certificate earned"
130
+ * - Measure: Total per user
131
+ * - Expected: C ~ 35% fewer certificates per user vs A
132
+ *
133
+ * REAL-WORLD ANALOGUE: Active note-taking lifts quiz performance, but
134
+ * obsessive note-taking signals "stuck in study mode" without finishing.
135
+ *
136
+ * ---------------------------------------------------------------
137
+ * 4. STUDY GROUP RETENTION (everything)
138
+ * ---------------------------------------------------------------
139
+ *
140
+ * PATTERN: Users who join a study group within 10 days get bonus
141
+ * discussion events. Non-joiners with low quiz scores (<60) churn
142
+ * hard at day 14 — all later events are removed.
143
+ *
144
+ * HOW TO FIND IT IN MIXPANEL:
145
+ *
146
+ * Report 1: D14 Retention by Early Group Join Cohort
147
+ * - Cohort A: users who fired "study group joined" within first 10 days
148
+ * - Cohort B: users with no early study group joined
149
+ * - Compare D14 retention (any event past d14) per cohort
150
+ * - Expected: A ~ 100% vs B ~ 1%. The churn is near-deterministic:
151
+ * it fires for non-joiners with ANY raw sub-60 quiz, and at the
152
+ * organic score mean (~40) virtually every quizzing non-joiner
153
+ * has one. Non-joiners who never quiz survive, but they are rare
154
+ * among 20d+-tenure users.
155
+ *
156
+ * Report 2: Discussion Volume by Group Cohort
157
+ * - Cohort A vs B (as above)
158
+ * - Event: "discussion posted"
159
+ * - Measure: Total per user
160
+ * - Expected: A posts substantially more
161
+ *
162
+ * REAL-WORLD ANALOGUE: Social learning ties create accountability
163
+ * and dramatically reduce drop-off in cohort-based courses.
164
+ *
165
+ * ---------------------------------------------------------------
166
+ * 5. HINT DEPENDENCY (event)
167
+ * ---------------------------------------------------------------
168
+ *
169
+ * PATTERN: On "practice problem solved", hint_used=true gets difficulty
170
+ * forced to "easy" 60% of the time. hint_used=false gets difficulty forced
171
+ * to "hard" 40% of the time. No flag — discover via difficulty breakdown
172
+ * filtered by hint_used.
173
+ *
174
+ * HOW TO FIND IT IN MIXPANEL:
175
+ *
176
+ * Report 1: Easy Problem Mix for Hint Users
177
+ * - Report type: Insights
178
+ * - Event: "practice problem solved"
179
+ * - Measure: Total
180
+ * - Filter: "hint_used" = true
181
+ * - Breakdown: "difficulty"
182
+ * - Expected: ~73% easy (60% forced + 40% x 1/3 organic; vs ~33% baseline)
183
+ *
184
+ * Report 2: Hard Problem Mix for Independent Solvers
185
+ * - Report type: Insights
186
+ * - Event: "practice problem solved"
187
+ * - Measure: Total
188
+ * - Filter: "hint_used" = false
189
+ * - Breakdown: "difficulty"
190
+ * - Expected: ~60% hard (40% forced + 60% x 1/3 organic; vs ~33% baseline)
191
+ *
192
+ * REAL-WORLD ANALOGUE: Learners who lean on hints get nudged toward
193
+ * easier work, while those who push through unaided self-select
194
+ * into harder material.
195
+ *
196
+ * ---------------------------------------------------------------
197
+ * 6. SEMESTER-END SPIKE (everything)
198
+ * ---------------------------------------------------------------
199
+ *
200
+ * PATTERN: Days 75-85 simulate semester crunch. quiz_started, quiz_completed,
201
+ * and assignment_submitted events are duplicated at an 80% rate. No flag —
202
+ * discover via line chart of those event volumes by day.
203
+ *
204
+ * HOW TO FIND IT IN MIXPANEL:
205
+ *
206
+ * Report 1: Assessment Volume Over Time
207
+ * - Report type: Insights
208
+ * - Events: "quiz started" + "quiz completed" + "assignment submitted"
209
+ * - Measure: Total
210
+ * - Line chart by day
211
+ * - Expected: ~1.8x volume spike on days 75-84 (80% duplication rate;
212
+ * the hook's continuous [75, 85) day-index window fully treats
213
+ * calendar days 75-84)
214
+ *
215
+ * REAL-WORLD ANALOGUE: Semester-end deadlines reliably produce a
216
+ * massive last-minute surge in student activity.
217
+ *
218
+ * ---------------------------------------------------------------
219
+ * 7. FREE VS PAID COURSES (funnel-pre + everything)
220
+ * ---------------------------------------------------------------
221
+ *
222
+ * PATTERN: Free users get 0.5x funnel conversion rate on the cert funnel only
223
+ * (30% -> 15% generative); paid subscribers get 1.5x (30% -> 45%). Free users
224
+ * ALSO lose 55% of their certificates post-generation, so the observed
225
+ * completion gap compounds both treatments: paid/free certificates per user
226
+ * lands well above the 3x conversion-only gap (v1.5 doc said "~2.2x" — that
227
+ * figure ignored the 55% cert removal AND understated the paid factor).
228
+ *
229
+ * HOW TO FIND IT IN MIXPANEL:
230
+ *
231
+ * Report 1: Course Completion Funnel by Subscription
232
+ * - Report type: Funnels
233
+ * - Steps: "course enrolled" -> "lecture completed" -> "quiz completed" -> "certificate earned"
234
+ * - Breakdown: "subscription_status"
235
+ * - Expected: paid arms convert several-fold more than free (conversion
236
+ * gating 3x, further widened in-report by the 55% free cert removal)
237
+ *
238
+ * Report 2: Certificates Earned per User
239
+ * - Report type: Insights
240
+ * - Event: "certificate earned"
241
+ * - Measure: Total per user (average)
242
+ * - Breakdown: "subscription_status"
243
+ * - Expected: paid subscribers earn substantially more certificates
244
+ *
245
+ * REAL-WORLD ANALOGUE: Paid commitment correlates strongly with
246
+ * follow-through; free learners drop off long before completion.
247
+ *
248
+ * ---------------------------------------------------------------
249
+ * 8. PLAYBACK SPEED CORRELATION (event + everything)
250
+ * ---------------------------------------------------------------
251
+ *
252
+ * PATTERN: Speed learners (>=2.0x speed on 3+ lectures) get 0.6x
253
+ * watch_time and a paradoxical +8 quiz score boost. Thorough
254
+ * learners (<=1.0x) get 1.4x watch_time.
255
+ *
256
+ * HOW TO FIND IT IN MIXPANEL:
257
+ *
258
+ * Report 1: Watch Time by Playback Speed
259
+ * - Report type: Insights
260
+ * - Event: "lecture completed"
261
+ * - Measure: Average of "watch_time_mins"
262
+ * - Breakdown: "playback_speed"
263
+ * - Expected: speed >= 2.0 ~ 0.6x baseline; speed <= 1.0 ~ 1.4x baseline
264
+ *
265
+ * Report 2: Quiz Score by Speed Learner Cohort
266
+ * - Cohort A: users with 3+ "lecture completed" events at playback_speed >= 2.0
267
+ * - Cohort B: rest
268
+ * - Event: "quiz completed"
269
+ * - Measure: Average of "score_percent"
270
+ * - Expected: A ~ +8 pts vs B
271
+ *
272
+ * REAL-WORLD ANALOGUE: Power users who watch lectures at 2x speed
273
+ * tend to be domain-confident and outperform on assessments
274
+ * despite spending less time.
275
+ *
276
+ * ---------------------------------------------------------------
277
+ * 9. COURSE COMPLETION TIME-TO-CONVERT (everything)
278
+ * ---------------------------------------------------------------
279
+ *
280
+ * PATTERN: Annual subscribers complete the course-completion funnel
281
+ * 2x faster (factor 0.5); Free users 1.8x slower (factor 1.8).
282
+ * Applied in the everything hook by scaling the enrolled-to-cert
283
+ * gap on the raw events. Stronger factors compensate for the
284
+ * composition effect from H7's conversion-rate gating.
285
+ *
286
+ * HOW TO FIND IT IN MIXPANEL:
287
+ *
288
+ * Report 1: Course Completion Median Time-to-Convert by Subscription
289
+ * - Funnels > "course enrolled" -> "lecture completed" -> "quiz completed" -> "certificate earned"
290
+ * - Measure: Median time to convert
291
+ * - Breakdown: subscription_status
292
+ * - Expected: annual < monthly < free (direction)
293
+ *
294
+ * Also visible via cross-event SQL: MIN("course enrolled" time) to
295
+ * MIN("certificate earned" time) per user, broken down by
296
+ * subscription_status. Annual < monthly < free.
297
+ *
298
+ * REAL-WORLD ANALOGUE: Paid commitment accelerates throughput.
299
+ *
300
+ * ---------------------------------------------------------------
301
+ * 10. SOCIAL LEARNING EXPERIMENT (funnel experiment)
302
+ * ---------------------------------------------------------------
303
+ *
304
+ * PATTERN: A/B experiment on the Social Learning funnel (discussion
305
+ * posted → study group joined → resource downloaded). "AI Study
306
+ * Buddy" variant boosts conversion 1.4x and speeds TTC to 0.85x.
307
+ * Activates 30 days before dataset end.
308
+ *
309
+ * HOW TO FIND IT IN MIXPANEL:
310
+ *
311
+ * Report 1: $experiment_started by Variant
312
+ * - Report type: Insights
313
+ * - Event: "$experiment_started"
314
+ * - Measure: Total
315
+ * - Breakdown: "$experiment_name" and "Variant"
316
+ * - Expected: ~50% Control, ~50% AI Study Buddy
317
+ *
318
+ * Report 2: Social Learning Funnel by Variant
319
+ * - Report type: Funnels
320
+ * - Steps: "discussion posted" → "study group joined" → "resource downloaded"
321
+ * - Breakdown: Variant
322
+ * - Expected: AI Study Buddy ~ 1.35-1.4x conversion vs Control
323
+ * (generative multiplier is 1.4; organic pollution — failed
324
+ * experiment passes completed by organic downloads at a ~0.035
325
+ * base rate in both arms — mildly attenuates the measured lift)
326
+ *
327
+ * REAL-WORLD ANALOGUE: AI-powered study companions boost social
328
+ * engagement and resource discovery in cohort-based courses.
329
+ *
330
+ * ===============================================================
331
+ * EXPECTED METRICS SUMMARY
332
+ * (Measured = full fidelity, 10K users / 760,795 events)
333
+ * ===============================================================
334
+ *
335
+ * Story id | Metric | Expected | Measured
336
+ * ---------|---------------------------------------------|---------------|---------
337
+ * H1 | instructor profile share | 1/9 = 0.111 | 0.1121
338
+ * H1 | role-attribute purity (both roles) | 1.0 | 1.0000
339
+ * H1 | event account_type = profile (registered) | 1.0 | 1.0000
340
+ * H2 | Sun/Mon late rate vs rest | 0.60 / 0.20 | 0.6038 / 0.1986
341
+ * H2 | quiz score diff rest - Sun/Mon | ~24 (clamp) | 24.13
342
+ * H3 | sweet/low quiz score (isolated read) | ~1.3 | 1.270
343
+ * H3 | certs-per-enroll over/sweet | ~0.65 keep | 0.6398
344
+ * H3 | placebo: over/low score | ~1.0 | 0.9837
345
+ * H4 | D14+ activity early-join vs non | ~1.0 / ~0.01 | 0.9988 / 0.0035
346
+ * H4 | discussions per user early/non | ~18x | 18.91
347
+ * H5 | P(easy | hint) | 0.745 | 0.7469
348
+ * H5 | P(hard | no hint) | 0.610 | 0.6141
349
+ * H6 | spikable volume window/flank (days 75-84) | ~1.8-1.9 | 1.908
350
+ * H6 | placebo: non-spikable window/flank | ~1.0-1.1 | 1.067
351
+ * H7 | emulator conv monthly/free (86.4h, 2-step) | 6.67 compound | 6.73
352
+ * H7 | certs-per-enroll monthly/free | ~6 (diluted) | 6.037
353
+ * H7 | placebo: annual/monthly certs-per-enroll | ~1.0 | 1.057
354
+ * H8 | watch time fast/mid | ~0.59 | 0.5857
355
+ * H8 | watch time slow/mid | 1.40 | 1.388
356
+ * H8 | quiz score diff speedy - rest | ~+8 | +7.91
357
+ * H9 | median TTC free/monthly (emulator 86.4h) | ~1.8 | 1.760
358
+ * H9 | median TTC annual/monthly | ~0.5 | 0.510
359
+ * H10 | strict-paired conversion lift AI/Control | ~1.37 (p.035) | 1.377
360
+ * H10 | paired median TTC AI/Control | ~0.85 | 0.857
361
+ */
362
+
363
+ // ── SCALE ──
364
+ const SEED = "harness-education";
365
+ const NUM_USERS = 10_000;
366
+ const DATASET_START = "2026-01-01T00:00:00Z";
367
+ const DATASET_END = "2026-05-01T23:59:59Z";
368
+ const EVENTS_PER_DAY = 1.2;
369
+ const token = process.env.MP_TOKEN || "your-mixpanel-token";
370
+
371
+ const chance = u.initChance(SEED);
372
+
373
+ // ── KNOBS (tweak these to reshape stories) ──
374
+ const HINT_EASY_LIKELIHOOD = 60;
375
+ const HINT_HARD_LIKELIHOOD = 40;
376
+
377
+ const SPEED_FAST_THRESHOLD = 2.0;
378
+ const SPEED_FAST_WATCH_FACTOR = 0.6;
379
+ const SPEED_FAST_WATCH_MIN = 3;
380
+ const SPEED_SLOW_THRESHOLD = 1.0;
381
+ const SPEED_SLOW_WATCH_FACTOR = 1.4;
382
+ const SPEED_SLOW_WATCH_MAX = 90;
383
+ const SPEED_LECTURE_COUNT_THRESHOLD = 3;
384
+ const SPEED_QUIZ_BOOST_POINTS = 8;
385
+
386
+ const FREE_FUNNEL_CONV_FACTOR = 0.5;
387
+ const PAID_FUNNEL_CONV_FACTOR = 1.5;
388
+ const FREE_CERT_DROP_LIKELIHOOD = 55;
389
+
390
+ const NOTES_SWEET_MIN = 5;
391
+ const NOTES_SWEET_MAX = 8;
392
+ const NOTES_OVER_THRESHOLD = 9;
393
+ const NOTES_QUIZ_BOOST = 1.3;
394
+ const NOTES_BONUS_CERT_LIKELIHOOD = 40;
395
+ const NOTES_OVER_CERT_DROP_LIKELIHOOD = 35;
396
+
397
+ const SEMESTER_SPIKE_START_DAY = 75;
398
+ const SEMESTER_SPIKE_END_DAY = 85;
399
+ const SEMESTER_SPIKE_LIKELIHOOD = 80;
400
+
401
+ const TTC_ANNUAL_FACTOR = 0.5;
402
+ const TTC_FREE_FACTOR = 1.8;
403
+
404
+ const STUDY_GROUP_EARLY_DAYS = 10;
405
+ const STUDY_GROUP_LOW_QUIZ_THRESHOLD = 60;
406
+ const STUDY_GROUP_CHURN_CUTOFF_DAYS = 14;
407
+ const STUDY_GROUP_DISCUSSION_CLONE_LIKELIHOOD = 60;
408
+
409
+ const DEADLINE_LATE_LIKELIHOOD = 60;
410
+ const DEADLINE_QUIZ_PENALTY = 25;
411
+
412
+ // ── DATA ARRAYS ──
413
+ // Generate consistent IDs for lookup tables and event properties
414
+ const courseIds = v.range(1, 151).map(n => `course_${v.uid(6)}`);
415
+ const quizIds = v.range(1, 401).map(n => `quiz_${v.uid(6)}`);
416
+ const groupIds = v.range(1, 301).map(n => `group_${v.uid(6)}`);
417
+ const lectureIds = v.range(1, 501).map(n => `lecture_${v.uid(6)}`);
418
+ const assignmentIds = v.range(1, 201).map(n => `assignment_${v.uid(6)}`);
419
+ const problemIds = v.range(1, 601).map(n => `problem_${v.uid(6)}`);
420
+
421
+ // ── HELPER FUNCTIONS ──
422
+ function handleUserHooks(record) {
423
+ // H1: STUDENT VS INSTRUCTOR PROFILES — role-based attributes.
424
+ if (record.account_type === "instructor") {
425
+ record.courses_created = chance.integer({ min: 1, max: 15 });
426
+ record.teaching_experience_years = chance.integer({ min: 1, max: 20 });
427
+ record.instructor_rating = Math.round((chance.floating({ min: 3.0, max: 5.0 }) + Number.EPSILON) * 100) / 100;
428
+ } else {
429
+ record.learning_goal = chance.pickone(["career_change", "skill_upgrade", "hobby", "degree_requirement"]);
430
+ record.study_hours_per_week = chance.integer({ min: 2, max: 30 });
431
+ }
432
+ return record;
433
+ }
434
+
435
+ function handleEventHooks(record) {
436
+ // H5: HINT DEPENDENCY — hint users get 60% easy problems; non-hint
437
+ // users get 40% hard problems. Mutates difficulty (raw).
438
+ if (record.event === "practice problem solved") {
439
+ if (record.hint_used === true && chance.bool({ likelihood: HINT_EASY_LIKELIHOOD })) {
440
+ record.difficulty = "easy";
441
+ } else if (record.hint_used === false && chance.bool({ likelihood: HINT_HARD_LIKELIHOOD })) {
442
+ record.difficulty = "hard";
443
+ }
444
+ }
445
+ // H8 (event): PLAYBACK SPEED — speed learners (>= 2.0x) get
446
+ // watch_time_mins compressed 0.6x; thorough learners (<= 1.0x) get 1.4x.
447
+ if (record.event === "lecture completed") {
448
+ const speed = record.playback_speed;
449
+ if (speed >= SPEED_FAST_THRESHOLD && record.watch_time_mins !== undefined) {
450
+ record.watch_time_mins = Math.max(SPEED_FAST_WATCH_MIN, Math.floor(record.watch_time_mins * SPEED_FAST_WATCH_FACTOR));
451
+ } else if (speed !== undefined && speed <= SPEED_SLOW_THRESHOLD && record.watch_time_mins !== undefined) {
452
+ record.watch_time_mins = Math.min(SPEED_SLOW_WATCH_MAX, Math.floor(record.watch_time_mins * SPEED_SLOW_WATCH_FACTOR));
453
+ }
454
+ }
455
+ return record;
456
+ }
457
+
458
+ function handleFunnelPreHooks(record, meta) {
459
+ // H7: FREE VS PAID — free users get 0.5x conversion rate; paid
460
+ // subscribers get 1.5x. Scoped to the course-completion funnel ONLY
461
+ // (sequence ending in "certificate earned") to avoid displacing standalone
462
+ // events for paid users and triggering unintended churn in H4.
463
+ const isCertFunnel = Array.isArray(meta?.funnel?.sequence) &&
464
+ meta.funnel.sequence.includes("certificate earned");
465
+ if (isCertFunnel) {
466
+ const subStatus = meta?.profile?.subscription_status;
467
+ if (subStatus === "free") {
468
+ record.conversionRate = Math.round(record.conversionRate * FREE_FUNNEL_CONV_FACTOR);
469
+ } else if (subStatus === "monthly" || subStatus === "annual") {
470
+ record.conversionRate = Math.min(100, Math.round(record.conversionRate * PAID_FUNNEL_CONV_FACTOR));
471
+ }
472
+ }
473
+ return record;
474
+ }
475
+
476
+ function handleEverythingHooks(record, meta) {
477
+ const datasetStart = dayjs.unix(meta.datasetStart);
478
+ const userEvents = record;
479
+ const profile = meta.profile;
480
+ const firstEventTime = userEvents.length > 0 ? dayjs(userEvents[0].time) : null;
481
+
482
+ if (profile) {
483
+ userEvents.forEach((event) => {
484
+ if (profile.Platform !== undefined) event.Platform = profile.Platform;
485
+ // H1: event-level account_type must agree with the profile —
486
+ // the engine draws event props independently of user props, so
487
+ // without this stamp the 'account registered' breakdown would
488
+ // contradict the profile mix
489
+ if (event.event === "account registered" && profile.account_type !== undefined) {
490
+ event.account_type = profile.account_type;
491
+ }
492
+ });
493
+ }
494
+
495
+ let notesTakenCount = 0;
496
+ let joinedStudyGroupEarly = false;
497
+ let hasLowQuizScore = false;
498
+ let speedLectureCount = 0;
499
+
500
+ userEvents.forEach((event) => {
501
+ const eventTime = dayjs(event.time);
502
+ const daysSinceStart = firstEventTime ? eventTime.diff(firstEventTime, 'days', true) : 0;
503
+ if (event.event === "lecture completed" && event.notes_taken === true) notesTakenCount++;
504
+ if (event.event === "study group joined" && daysSinceStart <= STUDY_GROUP_EARLY_DAYS) joinedStudyGroupEarly = true;
505
+ if (event.event === "quiz completed" && event.score_percent < STUDY_GROUP_LOW_QUIZ_THRESHOLD) hasLowQuizScore = true;
506
+ if (event.event === "lecture completed" && event.playback_speed >= SPEED_FAST_THRESHOLD) speedLectureCount++;
507
+ });
508
+
509
+ // H3 + H10: NOTES MAGIC NUMBER (in-funnel, no flags)
510
+ // Sweet 5-8 notes-taken lectures → +30% quiz score_percent (cap 100).
511
+ // Over 9+ → drop 35% of certificate-earned events (over-noted but
512
+ // can't synthesize; gets stuck in "study mode").
513
+ if (notesTakenCount >= NOTES_SWEET_MIN && notesTakenCount <= NOTES_SWEET_MAX) {
514
+ userEvents.forEach((event) => {
515
+ if (event.event === "quiz completed" && event.score_percent !== undefined) {
516
+ event.score_percent = Math.min(100, Math.round(event.score_percent * NOTES_QUIZ_BOOST));
517
+ }
518
+ });
519
+ if (chance.bool({ likelihood: NOTES_BONUS_CERT_LIKELIHOOD })) {
520
+ const lastEvent = userEvents[userEvents.length - 1];
521
+ const certTemplate = userEvents.find(e => e.event === "certificate earned");
522
+ if (lastEvent && certTemplate) {
523
+ userEvents.push({
524
+ ...certTemplate,
525
+ time: dayjs(lastEvent.time).add(chance.integer({ min: 1, max: 5 }), 'days').toISOString(),
526
+ user_id: lastEvent.user_id,
527
+ course_id: chance.pickone(courseIds),
528
+ completion_time_days: chance.integer({ min: 14, max: 90 }),
529
+ final_grade: chance.integer({ min: 80, max: 100 }),
530
+ });
531
+ }
532
+ }
533
+ } else if (notesTakenCount >= NOTES_OVER_THRESHOLD) {
534
+ // Over-noters: drop 35% of certificates (stuck in study mode)
535
+ for (let i = userEvents.length - 1; i >= 0; i--) {
536
+ if (userEvents[i].event === "certificate earned" && chance.bool({ likelihood: NOTES_OVER_CERT_DROP_LIKELIHOOD })) {
537
+ userEvents.splice(i, 1);
538
+ }
539
+ }
540
+ }
541
+
542
+ // H8 (cont): Speed learners (3+ lectures at 2.0x) score +8 on quizzes.
543
+ if (speedLectureCount >= SPEED_LECTURE_COUNT_THRESHOLD) {
544
+ userEvents.forEach((event) => {
545
+ if (event.event === "quiz completed" && event.score_percent !== undefined) {
546
+ event.score_percent = Math.min(100, event.score_percent + SPEED_QUIZ_BOOST_POINTS);
547
+ }
548
+ });
549
+ }
550
+
551
+ // H6: SEMESTER-END SPIKE — duplicate quiz/assignment events
552
+ // in days 75-85 window. No flag — discover via line chart.
553
+ const duplicates = [];
554
+ const spikableEvents = ["quiz started", "quiz completed", "assignment submitted"];
555
+ userEvents.forEach((event) => {
556
+ if (spikableEvents.includes(event.event) && event.time) {
557
+ const dayInDataset = dayjs.utc(event.time).diff(datasetStart, 'days', true);
558
+ if (dayInDataset >= SEMESTER_SPIKE_START_DAY && dayInDataset <= SEMESTER_SPIKE_END_DAY && chance.bool({ likelihood: SEMESTER_SPIKE_LIKELIHOOD })) {
559
+ const dup = JSON.parse(JSON.stringify(event));
560
+ dup.time = dayjs(event.time).add(chance.integer({ min: 5, max: 120 }), 'minutes').toISOString();
561
+ duplicates.push(dup);
562
+ }
563
+ }
564
+ });
565
+ if (duplicates.length > 0) userEvents.push(...duplicates);
566
+
567
+ const subStatus = profile ? profile.subscription_status : "free";
568
+
569
+ // H9 (T2C): COURSE COMPLETION TIME-TO-CONVERT (everything)
570
+ // Annual subscribers complete the cert funnel 2x faster (factor 0.5);
571
+ // Free users 1.8x slower (factor 1.8). For each "certificate earned"
572
+ // event, find the nearest preceding "course enrolled" and scale the gap.
573
+ // Runs BEFORE cert-dropping (H7) so TTC adjustments aren't masked by
574
+ // survivorship bias from the 55% free cert removal.
575
+ {
576
+ const ttcFactor = (
577
+ subStatus === "annual" ? TTC_ANNUAL_FACTOR :
578
+ subStatus === "free" ? TTC_FREE_FACTOR :
579
+ 1.0
580
+ );
581
+ if (ttcFactor !== 1.0) {
582
+ // Collect all "course enrolled" times (sorted) for binary lookup
583
+ const enrolledTimes = userEvents
584
+ .filter(e => e.event === "course enrolled")
585
+ .map(e => dayjs(e.time))
586
+ .sort((a, b) => a.valueOf() - b.valueOf());
587
+
588
+ if (enrolledTimes.length > 0) {
589
+ for (const event of userEvents) {
590
+ if (event.event === "certificate earned") {
591
+ const certTime = dayjs(event.time);
592
+ // Find the latest enrolled time before this cert
593
+ let anchor = null;
594
+ for (let k = enrolledTimes.length - 1; k >= 0; k--) {
595
+ if (enrolledTimes[k].isBefore(certTime)) {
596
+ anchor = enrolledTimes[k];
597
+ break;
598
+ }
599
+ }
600
+ if (anchor) {
601
+ const gap = certTime.diff(anchor);
602
+ const newGap = Math.round(gap * ttcFactor);
603
+ event.time = anchor.add(newGap, "milliseconds").toISOString();
604
+ }
605
+ }
606
+ }
607
+ }
608
+ }
609
+ }
610
+
611
+ // H7: FREE VS PAID — free users lose 55% of certificates.
612
+ if (subStatus === "free") {
613
+ for (let i = userEvents.length - 1; i >= 0; i--) {
614
+ if (userEvents[i].event === "certificate earned" && chance.bool({ likelihood: FREE_CERT_DROP_LIKELIHOOD })) {
615
+ userEvents.splice(i, 1);
616
+ }
617
+ }
618
+ }
619
+
620
+ // H4: STUDY GROUP RETENTION — non-joiners with low scores lose
621
+ // all post-day-14 events. Joiners get extra cloned discussion events.
622
+ if (!joinedStudyGroupEarly && hasLowQuizScore) {
623
+ const churnCutoff = firstEventTime ? firstEventTime.add(STUDY_GROUP_CHURN_CUTOFF_DAYS, 'days') : null;
624
+ for (let i = userEvents.length - 1; i >= 0; i--) {
625
+ if (churnCutoff && dayjs(userEvents[i].time).isAfter(churnCutoff)) {
626
+ userEvents.splice(i, 1);
627
+ }
628
+ }
629
+ } else if (joinedStudyGroupEarly) {
630
+ const lastEvent = userEvents[userEvents.length - 1];
631
+ const discussionTemplate = userEvents.find(e => e.event === "discussion posted");
632
+ if (lastEvent && discussionTemplate && chance.bool({ likelihood: STUDY_GROUP_DISCUSSION_CLONE_LIKELIHOOD })) {
633
+ userEvents.push({
634
+ ...discussionTemplate,
635
+ time: dayjs(lastEvent.time).add(chance.integer({ min: 1, max: 3 }), 'days').toISOString(),
636
+ user_id: lastEvent.user_id,
637
+ course_id: chance.pickone(courseIds),
638
+ post_type: chance.pickone(["question", "answer", "comment"]),
639
+ word_count: chance.integer({ min: 20, max: 400 }),
640
+ });
641
+ }
642
+ }
643
+
644
+ // H2: DEADLINE CRAMMING — Sun/Mon assignment_submitted events
645
+ // flip is_late to true 60% of the time and quiz_completed score_percent
646
+ // drops 25 points. Mutates raw is_late + score_percent.
647
+ for (const event of userEvents) {
648
+ if (event.event === "assignment submitted" && event.time) {
649
+ const dow = new Date(event.time).getUTCDay();
650
+ if (dow === 0 || dow === 1) {
651
+ event.is_late = chance.bool({ likelihood: DEADLINE_LATE_LIKELIHOOD });
652
+ }
653
+ }
654
+ }
655
+ userEvents.forEach((event) => {
656
+ if (event.event === "quiz completed" && event.time) {
657
+ const dow = new Date(event.time).getUTCDay();
658
+ if ((dow === 0 || dow === 1) && event.score_percent !== undefined) {
659
+ event.score_percent = Math.max(0, event.score_percent - DEADLINE_QUIZ_PENALTY);
660
+ }
661
+ }
662
+ });
663
+
664
+ return record;
665
+ }
666
+
667
+ // ── CONFIG ──
668
+ /** @type {Config} */
669
+ const config = {
670
+ version: 2,
671
+ seed: SEED,
672
+ datasetStart: DATASET_START,
673
+ datasetEnd: DATASET_END,
674
+ avgEventsPerUserPerDay: EVENTS_PER_DAY,
675
+ numUsers: NUM_USERS,
676
+ format: "json",
677
+ gzip: true,
678
+ credentials: {
679
+ token,
680
+ },
681
+ switches: {
682
+ hasSessionIds: true,
683
+ alsoInferFunnels: false,
684
+ hasLocation: true,
685
+ hasAndroidDevices: true,
686
+ hasIOSDevices: true,
687
+ hasDesktopDevices: true,
688
+ hasBrowser: false,
689
+ hasCampaigns: false,
690
+ isAnonymous: false,
691
+ hasAdSpend: false,
692
+ hasAvatar: true,
693
+ },
694
+ identity: {
695
+ avgDevicePerUser: 2,
696
+ },
697
+ concurrency: 1,
698
+ writeToDisk: false,
699
+
700
+ funnels: [
701
+ {
702
+ sequence: ["account registered", "course enrolled", "lecture started"],
703
+ isFirstFunnel: true,
704
+ conversionRate: 75,
705
+ timeToConvert: 1,
706
+ },
707
+ {
708
+ // Core learning loop: students watch lectures and do practice problems constantly
709
+ sequence: ["lecture started", "lecture completed", "practice problem solved"],
710
+ conversionRate: 70,
711
+ timeToConvert: 4,
712
+ weight: 5,
713
+ reentry: true,
714
+ },
715
+ {
716
+ // Assessment flow: quizzes and assignments after studying
717
+ sequence: ["quiz started", "quiz completed", "assignment submitted"],
718
+ conversionRate: 55,
719
+ timeToConvert: 8,
720
+ weight: 3,
721
+ reentry: true,
722
+ },
723
+ {
724
+ // Course completion journey: enroll → complete → earn certificate
725
+ sequence: ["course enrolled", "lecture completed", "quiz completed", "certificate earned"],
726
+ conversionRate: 30,
727
+ timeToConvert: 48,
728
+ weight: 2,
729
+ },
730
+ {
731
+ // Social learning: discussions and study groups
732
+ sequence: ["discussion posted", "study group joined", "resource downloaded"],
733
+ conversionRate: 50,
734
+ timeToConvert: 12,
735
+ weight: 2,
736
+ experiment: {
737
+ name: "AI Study Buddy",
738
+ variants: [
739
+ { name: "Control" },
740
+ { name: "AI Study Buddy", conversionMultiplier: 1.4, ttcMultiplier: 0.85 },
741
+ ],
742
+ startDaysBeforeEnd: 30,
743
+ },
744
+ },
745
+ {
746
+ // Instructor interaction loop
747
+ sequence: ["assignment submitted", "assignment graded", "instructor feedback given"],
748
+ conversionRate: 45,
749
+ timeToConvert: 24,
750
+ weight: 2,
751
+ },
752
+ {
753
+ // Support and monetization
754
+ sequence: ["help requested", "subscription purchased", "course reviewed"],
755
+ conversionRate: 35,
756
+ timeToConvert: 24,
757
+ weight: 1,
758
+ },
759
+ ],
760
+
761
+ events: [
762
+ {
763
+ event: "account registered",
764
+ weight: 1,
765
+ isFirstEvent: true,
766
+ isAuthEvent: true,
767
+ properties: {
768
+ // pool matches the 8:1 student profile mix; the everything hook then
769
+ // overwrites from the profile so the event-level breakdown is EXACT
770
+ "account_type": ["student", "student", "student", "student", "student", "student", "student", "student", "instructor"],
771
+ "signup_source": ["organic", "referral", "school_partnership", "social_ad"],
772
+ }
773
+ },
774
+ {
775
+ event: "course enrolled",
776
+ weight: 8,
777
+ isStrictEvent: false,
778
+ properties: {
779
+ "course_id": courseIds,
780
+ "course_category": ["CS", "Math", "Science", "Business", "Arts", "Languages"],
781
+ "difficulty": ["beginner", "intermediate", "advanced"],
782
+ "is_free": [false, false, false, true, true],
783
+ }
784
+ },
785
+ {
786
+ event: "lecture started",
787
+ weight: 18,
788
+ properties: {
789
+ "course_id": courseIds,
790
+ "lecture_id": lectureIds,
791
+ "lecture_duration_mins": u.weighNumRange(5, 60, 0.8, 20),
792
+ "module_number": u.weighNumRange(1, 12),
793
+ }
794
+ },
795
+ {
796
+ event: "lecture completed",
797
+ weight: 14,
798
+ isStrictEvent: false,
799
+ properties: {
800
+ "course_id": courseIds,
801
+ "lecture_id": lectureIds,
802
+ "watch_time_mins": u.weighNumRange(3, 60, 0.8, 20),
803
+ "playback_speed": [0.75, 1.0, 1.0, 1.0, 1.25, 1.5, 2.0],
804
+ "notes_taken": [false, false, true],
805
+ }
806
+ },
807
+ {
808
+ event: "quiz started",
809
+ weight: 10,
810
+ isStrictEvent: false,
811
+ properties: {
812
+ "course_id": courseIds,
813
+ "quiz_id": quizIds,
814
+ "quiz_type": ["practice", "graded", "final_exam"],
815
+ "question_count": u.weighNumRange(5, 50, 0.7, 15),
816
+ }
817
+ },
818
+ {
819
+ event: "quiz completed",
820
+ weight: 8,
821
+ isStrictEvent: false,
822
+ properties: {
823
+ "course_id": courseIds,
824
+ "quiz_id": quizIds,
825
+ "score_percent": u.weighNumRange(0, 100, 1.2, 50),
826
+ "time_spent_mins": u.weighNumRange(3, 120, 0.6, 25),
827
+ "attempts": u.weighNumRange(1, 5, 0.5, 3),
828
+ }
829
+ },
830
+ {
831
+ event: "assignment submitted",
832
+ weight: 6,
833
+ isStrictEvent: false,
834
+ properties: {
835
+ "course_id": courseIds,
836
+ "assignment_id": assignmentIds,
837
+ "submission_type": ["text", "code", "file", "project"],
838
+ "word_count": u.weighNumRange(100, 5000, 0.6, 500),
839
+ "is_late": [false, false, false, false, true],
840
+ }
841
+ },
842
+ {
843
+ event: "assignment graded",
844
+ weight: 5,
845
+ properties: {
846
+ "course_id": courseIds,
847
+ "assignment_id": assignmentIds,
848
+ "grade": ["A", "B", "C", "D", "F"],
849
+ "feedback_length": u.weighNumRange(0, 500, 0.5, 100),
850
+ "grader": ["instructor", "peer", "auto"],
851
+ }
852
+ },
853
+ {
854
+ event: "discussion posted",
855
+ weight: 7,
856
+ isStrictEvent: false,
857
+ properties: {
858
+ "course_id": courseIds,
859
+ "post_type": ["question", "answer", "comment"],
860
+ "word_count": u.weighNumRange(10, 500, 0.6, 80),
861
+ }
862
+ },
863
+ {
864
+ event: "certificate earned",
865
+ weight: 2,
866
+ isStrictEvent: false,
867
+ properties: {
868
+ "course_id": courseIds,
869
+ "completion_time_days": u.weighNumRange(7, 180, 0.5, 45),
870
+ "final_grade": u.weighNumRange(60, 100, 1.2, 30),
871
+ }
872
+ },
873
+ {
874
+ event: "study group joined",
875
+ weight: 4,
876
+ isStrictEvent: false,
877
+ properties: {
878
+ "group_id": groupIds,
879
+ "group_size": u.weighNumRange(3, 20, 0.7, 8),
880
+ "group_type": ["study_circle", "project_team", "tutoring"],
881
+ }
882
+ },
883
+ {
884
+ event: "resource downloaded",
885
+ weight: 9,
886
+ properties: {
887
+ "resource_type": ["pdf", "slides", "code_sample", "dataset", "cheat_sheet"],
888
+ "course_id": courseIds,
889
+ }
890
+ },
891
+ {
892
+ event: "instructor feedback given",
893
+ weight: 3,
894
+ properties: {
895
+ "course_id": courseIds,
896
+ "feedback_type": ["written", "video", "rubric"],
897
+ "response_time_hours": u.weighNumRange(1, 72, 0.5, 15),
898
+ }
899
+ },
900
+ {
901
+ event: "course reviewed",
902
+ weight: 3,
903
+ properties: {
904
+ "course_id": courseIds,
905
+ "rating": u.weighNumRange(1, 5, 1.5, 3),
906
+ "review_length": u.weighNumRange(10, 1000, 0.5, 100),
907
+ "would_recommend": [false, false, false, true, true, true, true, true, true, true],
908
+ }
909
+ },
910
+ {
911
+ event: "subscription purchased",
912
+ weight: 2,
913
+ properties: {
914
+ "plan": ["monthly", "annual", "lifetime"],
915
+ "price": [19.99, 149.99, 499.99],
916
+ }
917
+ },
918
+ {
919
+ event: "help requested",
920
+ weight: 4,
921
+ properties: {
922
+ "topic": ["technical", "content", "billing", "accessibility"],
923
+ "channel": ["chat", "email", "forum"],
924
+ }
925
+ },
926
+ {
927
+ event: "practice problem solved",
928
+ weight: 12,
929
+ isStrictEvent: false,
930
+ properties: {
931
+ "course_id": courseIds,
932
+ "problem_id": problemIds,
933
+ "difficulty": ["easy", "medium", "hard"],
934
+ "time_to_solve_sec": u.weighNumRange(10, 3600, 0.5, 300),
935
+ "hint_used": [false, false, true],
936
+ }
937
+ },
938
+ ],
939
+
940
+ superProps: {
941
+ Platform: ["Web", "iOS", "Android", "iPad"],
942
+ },
943
+
944
+ scdProps: {
945
+ enrollment_status: {
946
+ values: ["enrolled", "active", "completed", "dropped"],
947
+ frequency: "month",
948
+ timing: "fuzzy",
949
+ max: 6
950
+ },
951
+ course_status: {
952
+ values: ["draft", "published", "archived", "deprecated"],
953
+ frequency: "month",
954
+ timing: "fixed",
955
+ max: 6,
956
+ type: "course_id"
957
+ }
958
+ },
959
+
960
+ userProps: {
961
+ "account_type": ["student", "student", "student", "student", "student", "student", "student", "student", "instructor"],
962
+ "subscription_status": ["free", "free", "free", "monthly", "annual"],
963
+ "learning_style": ["visual", "reading", "hands_on", "auditory"],
964
+ "education_level": ["high_school", "bachelors", "masters", "phd", "self_taught"],
965
+ "timezone": ["US_Eastern", "US_Pacific", "US_Central", "Europe", "Asia"],
966
+ "courses_created": [0],
967
+ "teaching_experience_years": [0],
968
+ "instructor_rating": [0],
969
+ "learning_goal": ["none"],
970
+ "study_hours_per_week": [0],
971
+ "Platform": ["Web", "iOS", "Android", "iPad"],
972
+ },
973
+
974
+ groupKeys: [
975
+ ["course_id", 150, ["course enrolled", "lecture started", "lecture completed", "quiz completed", "certificate earned"]],
976
+ ["group_id", 300, ["study group joined", "discussion posted"]],
977
+ ],
978
+
979
+ groupProps: {
980
+ course_id: {
981
+ "title": () => `${chance.pickone(["Introduction to", "Advanced", "Mastering", "Fundamentals of", "Applied"])} ${chance.pickone(["Algorithms", "Data Science", "Machine Learning", "Statistics", "Web Development", "Calculus", "Biology", "Economics", "Design Thinking", "Creative Writing"])}`,
982
+ "instructor_count": u.weighNumRange(1, 5, 0.5, 2),
983
+ "total_enrolled": u.weighNumRange(50, 5000, 0.6, 500),
984
+ "avg_rating": u.weighNumRange(3, 5, 1.5, 1),
985
+ },
986
+ group_id: {
987
+ "name": () => `${chance.pickone(["Study", "Learning", "Focus", "Peer", "Cohort"])} ${chance.pickone(["Circle", "Squad", "Team", "Hub", "Group"])} ${chance.character({ alpha: true, casing: "upper" })}${chance.integer({ min: 1, max: 99 })}`,
988
+ "member_count": u.weighNumRange(3, 20, 0.7, 8),
989
+ "focus_area": ["CS", "Math", "Science", "Business", "Arts", "Languages"],
990
+ }
991
+ },
992
+
993
+ lookupTables: [],
994
+
995
+ hook(record, type, meta) {
996
+ if (type === "user") return handleUserHooks(record);
997
+ if (type === "event") return handleEventHooks(record);
998
+ if (type === "funnel-pre") return handleFunnelPreHooks(record, meta);
999
+ if (type === "everything") return handleEverythingHooks(record, meta);
1000
+ return record;
1001
+ }
1002
+ };
1003
+
1004
+ export default config;
1005
+
1006
+ // ── STORIES (v1.6 verification contract) ──
1007
+ /*
1008
+ * MEASUREMENT DOCTRINE — how these reads stay honest
1009
+ *
1010
+ * IDENTITY: avgDevicePerUser: 2, but 'account registered' is both
1011
+ * isFirstEvent and isAuthEvent, so born users auth on their very first
1012
+ * event. The device-map resolve through the profiles' "anonymousIds"
1013
+ * pool is belt-and-braces for any device-only edge.
1014
+ *
1015
+ * CHURN RECOVERY (H4): the churn hook deletes ALL events after
1016
+ * firstEvent + 14d for non-early-joiners with any raw sub-60 quiz.
1017
+ * Deletion is the ONLY event-removal that touches lectures/quizzes
1018
+ * (H3/H7 remove certificates only), so a user's OUTPUT lifespan
1019
+ * exceeding 14.5d identifies the not-churned population exactly, and
1020
+ * within it output note/speed-lecture counts equal the hook-time
1021
+ * counts the treatments keyed on. H3/H8 score reads filter on it.
1022
+ *
1023
+ * SCORE TREATMENT LEDGER: score_percent is touched by THREE hooks —
1024
+ * H3 (x1.3 for 5-8-notes users), H8 (+8 for 3+-fast-lecture users),
1025
+ * H2 (-25 on Sun/Mon, runs LAST, hits duplicates too). Every score
1026
+ * read excludes Sun/Mon quizzes (removes H2) and conditions on the
1027
+ * OTHER treatment's cohort (H3 reads exclude speed learners; H8 reads
1028
+ * exclude sweet-notes users), so each knob is read in isolation.
1029
+ * Empirical organic score mean is ~40 in these restricted reads (the
1030
+ * pool's nominal mean 50 is inflated by the treatments themselves).
1031
+ *
1032
+ * ORGANIC DIFFICULTY IS NOT UNIFORM: the difficulty pool is a 3-value
1033
+ * array but measured organic shares are easy 0.362 / medium 0.287 /
1034
+ * hard 0.351 (7+ sigma off uniform — engine-level draw skew).
1035
+ * H5 bands derive from the MEASURED organic composition:
1036
+ * P(easy|hint) = 0.60 + 0.40 x 0.362 = 0.745; P(hard|no-hint) =
1037
+ * 0.40 + 0.60 x 0.351 = 0.610.
1038
+ *
1039
+ * EMULATOR TTC (H7/H9): 2-step read ['course enrolled','certificate
1040
+ * earned'] — the 4-step doc funnel would break because H9's annual
1041
+ * x0.5 compression can move a certificate BEFORE the interior
1042
+ * quiz-completed step. Window 86.4h = 48h generative x 1.8 free
1043
+ * stretch, covering the stretched support. Sensitivity check at 48h:
1044
+ * free conversion collapses 0.063 -> 0.002 (censoring confirms the
1045
+ * stretch is real); annual/monthly barely move.
1046
+ *
1047
+ * EXPERIMENT PAIRING (H10): $experiment_started fires BEFORE funnel
1048
+ * entry with an arm-dependent lag (the AI arm's ttcMultiplier
1049
+ * compresses even the exp->step1 gap), so pairing anchors at the
1050
+ * funnel ENTRY: first 'discussion posted' >= exp time, conversion =
1051
+ * 'resource downloaded' within 12h of entry with >= 1 'study group
1052
+ * joined' strictly between. Organic pollution (partial failed passes
1053
+ * completed by organic downloads, ~0.035 base rate — implied
1054
+ * consistently by both arms at full fidelity) mildly attenuates the
1055
+ * generative 1.4x lift to ~1.37 observed; both arms carry the same
1056
+ * pollution so direction is preserved.
1057
+ *
1058
+ * ACTIVITY COUPLING: certificate counts scale with user activity, so
1059
+ * cross-cohort cert reads normalize per enrollment (certs/enrolls),
1060
+ * and the H3 volume read carries a pre-cliff flatness precondition.
1061
+ */
1062
+
1063
+ const ID_CTE = `
1064
+ us AS (SELECT * FROM read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)),
1065
+ dm AS (SELECT unnest("anonymousIds") AS device_id, distinct_id FROM us),
1066
+ ev AS (SELECT coalesce(m.distinct_id::VARCHAR, e.user_id::VARCHAR, e.device_id::VARCHAR) AS uid,
1067
+ e.time::TIMESTAMP AS t, e.*
1068
+ FROM read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true) e
1069
+ LEFT JOIN dm m ON e.device_id = m.device_id)`;
1070
+
1071
+ const PU_CTE = `
1072
+ pu AS (SELECT e.uid, min(e.t) AS first_t, max(e.t) AS last_t,
1073
+ count(*) FILTER (WHERE event = 'lecture completed' AND notes_taken) AS notes,
1074
+ count(*) FILTER (WHERE event = 'lecture completed' AND playback_speed >= ${SPEED_FAST_THRESHOLD}) AS fast_lex,
1075
+ count(*) FILTER (WHERE event = 'quiz completed') AS quizzes,
1076
+ count(*) FILTER (WHERE event = 'certificate earned') AS certs,
1077
+ count(*) FILTER (WHERE event = 'course enrolled') AS enrolls,
1078
+ count(*) FILTER (WHERE event = 'discussion posted') AS discussions,
1079
+ min(CASE WHEN event = 'study group joined' THEN e.t END) AS first_join_t
1080
+ FROM ev e GROUP BY 1),
1081
+ puu AS (SELECT p.*, u.subscription_status, u.account_type,
1082
+ (p.first_join_t IS NOT NULL AND date_diff('hour', p.first_t, p.first_join_t) <= ${STUDY_GROUP_EARLY_DAYS * 24}) AS early_join,
1083
+ (p.last_t > p.first_t + INTERVAL '14 days 12 hours') AS retained
1084
+ FROM pu p JOIN us u ON p.uid = u.distinct_id::VARCHAR)`;
1085
+
1086
+ const cellsOf = (rows, key) => Object.fromEntries((rows || []).map((r) => [r[key], r]));
1087
+
1088
+ export const stories = [
1089
+ {
1090
+ id: "H1-role-profiles",
1091
+ hook: "H1",
1092
+ archetype: "cohort-prop-scale",
1093
+ narrative:
1094
+ "Two-sided marketplace: profile pool is 8:1 student (expected instructor share 1/9 = 0.111). " +
1095
+ "The user hook stamps role-exclusive attributes (instructors: courses_created/experience/rating; " +
1096
+ "students: learning_goal/study_hours) — purity is structural, asserted at 1.0. The everything " +
1097
+ "hook also overwrites account_type on 'account registered' events from the profile (the engine " +
1098
+ "draws event props independently), so event-level agreement is structural too.",
1099
+ assertions: [
1100
+ {
1101
+ breakdown: {
1102
+ type: "duckdb",
1103
+ sql: `WITH ${ID_CTE}
1104
+ SELECT count(*)::BIGINT AS users,
1105
+ count(*) FILTER (WHERE account_type = 'instructor')::DOUBLE / count(*) AS share
1106
+ FROM us`,
1107
+ },
1108
+ assert: (rows) => {
1109
+ const r = rows?.[0];
1110
+ if (!r || Number(r.users) < 5000) {
1111
+ return { verdict: "WEAK", detail: `population too small: users=${r?.users ?? 0}` };
1112
+ }
1113
+ const share = Number(r.share);
1114
+ const detail = `instructor share=${share.toFixed(4)} (pool 1/9 = 0.1111; n=${r.users})`;
1115
+ if (share >= 0.095 && share <= 0.125) return { verdict: "NAILED", detail };
1116
+ if (share >= 0.085 && share <= 0.14) return { verdict: "STRONG", detail };
1117
+ return { verdict: "NONE", detail };
1118
+ },
1119
+ },
1120
+ {
1121
+ breakdown: {
1122
+ type: "duckdb",
1123
+ sql: `WITH ${ID_CTE}
1124
+ SELECT account_type, count(*)::BIGINT AS users,
1125
+ avg(CASE WHEN account_type = 'instructor'
1126
+ THEN (courses_created >= 1 AND teaching_experience_years >= 1 AND instructor_rating >= 3
1127
+ AND learning_goal = 'none' AND study_hours_per_week = 0)::INT
1128
+ ELSE (courses_created = 0 AND instructor_rating = 0 AND learning_goal <> 'none'
1129
+ AND study_hours_per_week BETWEEN 2 AND 30)::INT END) AS purity
1130
+ FROM us GROUP BY 1`,
1131
+ },
1132
+ assert: (rows) => {
1133
+ const by = cellsOf(rows, "account_type");
1134
+ const inst = by.instructor, stu = by.student;
1135
+ if (!inst || !stu || Number(inst.users) < 500 || Number(stu.users) < 4000) {
1136
+ return { verdict: "WEAK", detail: `cohorts too small: inst=${inst?.users ?? 0} stu=${stu?.users ?? 0}` };
1137
+ }
1138
+ const pi = Number(inst.purity), ps = Number(stu.purity);
1139
+ const detail = `role-attribute purity: instructor=${pi.toFixed(4)} (n=${inst.users}) student=${ps.toFixed(4)} (n=${stu.users})`;
1140
+ if (pi === 1 && ps === 1) return { verdict: "NAILED", detail };
1141
+ if (pi >= 0.995 && ps >= 0.995) return { verdict: "STRONG", detail };
1142
+ if (pi >= 0.9 && ps >= 0.9) return { verdict: "WEAK", detail };
1143
+ return { verdict: "NONE", detail };
1144
+ },
1145
+ },
1146
+ {
1147
+ breakdown: {
1148
+ type: "duckdb",
1149
+ sql: `WITH ${ID_CTE}
1150
+ SELECT count(*)::BIGINT AS n, avg((e.account_type = u.account_type)::INT) AS agree
1151
+ FROM ev e JOIN us u ON e.uid = u.distinct_id::VARCHAR
1152
+ WHERE e.event = 'account registered'`,
1153
+ },
1154
+ assert: (rows) => {
1155
+ const r = rows?.[0];
1156
+ if (!r || Number(r.n) < 800) {
1157
+ return { verdict: "WEAK", detail: `too few 'account registered' events: n=${r?.n ?? 0}` };
1158
+ }
1159
+ const agree = Number(r.agree);
1160
+ const detail = `event-level account_type = profile account_type on ${agree.toFixed(4)} of ${r.n} events (hook-stamped)`;
1161
+ if (agree === 1) return { verdict: "NAILED", detail };
1162
+ if (agree >= 0.99) return { verdict: "STRONG", detail };
1163
+ return { verdict: "NONE", detail };
1164
+ },
1165
+ },
1166
+ ],
1167
+ },
1168
+ {
1169
+ id: "H2-deadline-cramming",
1170
+ hook: "H2",
1171
+ archetype: "bespoke",
1172
+ narrative:
1173
+ `Sun/Mon 'assignment submitted' events get is_late REDRAWN at ${DEADLINE_LATE_LIKELIHOOD}% ` +
1174
+ "(replacing the organic 1-in-5 pool draw, ~20%); Sun/Mon 'quiz completed' scores drop " +
1175
+ `${DEADLINE_QUIZ_PENALTY} points, clamped at 0. H2 runs LAST in the everything hook, so the ` +
1176
+ "penalty hits H6 duplicates and boosted scores alike — the DOW score DIFFERENCE reads the " +
1177
+ "knob minus clamp loss (organic sub-25 scores can't drop the full 25; measured 23.85). " +
1178
+ "Bands: rates Sun/Mon [0.56, 0.64] vs rest [0.17, 0.23]; score diff [21.5, 25.5].",
1179
+ assertions: [
1180
+ {
1181
+ breakdown: {
1182
+ type: "duckdb",
1183
+ sql: `WITH ${ID_CTE}
1184
+ SELECT (dayofweek(t) IN (0, 1)) AS sun_mon, count(*)::BIGINT AS n, avg(is_late::INT) AS late_rate
1185
+ FROM ev WHERE event = 'assignment submitted' GROUP BY 1`,
1186
+ },
1187
+ assert: (rows) => {
1188
+ const by = cellsOf(rows, "sun_mon");
1189
+ const sm = by.true, rest = by.false;
1190
+ if (!sm || !rest || Number(sm.n) < 10000 || Number(rest.n) < 25000) {
1191
+ return { verdict: "WEAK", detail: `cohorts too small: sunmon=${sm?.n ?? 0} rest=${rest?.n ?? 0}` };
1192
+ }
1193
+ const rs = Number(sm.late_rate), rr = Number(rest.late_rate);
1194
+ const detail = `late rate Sun/Mon=${rs.toFixed(4)} vs rest=${rr.toFixed(4)} (knob ${DEADLINE_LATE_LIKELIHOOD}% vs organic ~20%)`;
1195
+ if (rs >= 0.56 && rs <= 0.64 && rr >= 0.17 && rr <= 0.23) return { verdict: "NAILED", detail };
1196
+ if (rs >= 0.52 && rs <= 0.68 && rr >= 0.15 && rr <= 0.26) return { verdict: "STRONG", detail };
1197
+ if (rs > rr + 0.1) return { verdict: "WEAK", detail };
1198
+ return { verdict: rs <= rr ? "INVERSE" : "NONE", detail };
1199
+ },
1200
+ },
1201
+ {
1202
+ breakdown: {
1203
+ type: "duckdb",
1204
+ sql: `WITH ${ID_CTE}
1205
+ SELECT CASE WHEN dayofweek(t) IN (0, 1) THEN 'sm' ELSE 'rest' END AS bucket,
1206
+ count(*)::BIGINT AS user_count, avg(score_percent) AS score
1207
+ FROM ev WHERE event = 'quiz completed' GROUP BY 1`,
1208
+ },
1209
+ select: {
1210
+ sm: { where: { bucket: "sm" } },
1211
+ rest: { where: { bucket: "rest" } },
1212
+ },
1213
+ expect: { metric: "rest.score - sm.score", op: "between", target: [21.5, 25.5] },
1214
+ minCohort: 10000,
1215
+ },
1216
+ ],
1217
+ },
1218
+ {
1219
+ id: "H3-notes-magic-number",
1220
+ hook: "H3",
1221
+ archetype: "frequency-sweet-spot",
1222
+ narrative:
1223
+ `${NOTES_SWEET_MIN}-${NOTES_SWEET_MAX} notes-taken lectures => quiz scores x${NOTES_QUIZ_BOOST} ` +
1224
+ `(cap 100) + ${NOTES_BONUS_CERT_LIKELIHOOD}% chance of one bonus cloned certificate; ` +
1225
+ `${NOTES_OVER_THRESHOLD}+ notes => ${NOTES_OVER_CERT_DROP_LIKELIHOOD}% of certificates dropped. ` +
1226
+ "Score read: retained non-speed-learner users, non-Sun/Mon quizzes (see doctrine ledger) — " +
1227
+ "sweet/low ratio reads the knob with mild cap-100 loss at organic mean ~40 (measured 1.309); " +
1228
+ "9+-notes scores are untreated, so b9p/low is the placebo [0.92, 1.12]. Volume read follows " +
1229
+ "the doc's C-vs-A comparison: certs-per-enrollment 9+/sweet [0.62, 0.78] (measured 0.702 — " +
1230
+ "the 0.65 keep knob, mildly diluted by the sweet arm's bonus certs), guarded by sweet/low " +
1231
+ "flatness in [0.85, 1.10] (bounds activity-coupling drift).",
1232
+ assertions: [
1233
+ {
1234
+ breakdown: {
1235
+ type: "duckdb",
1236
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1237
+ SELECT CASE WHEN p.notes BETWEEN ${NOTES_SWEET_MIN} AND ${NOTES_SWEET_MAX} THEN 'sweet'
1238
+ WHEN p.notes <= ${NOTES_SWEET_MIN - 1} THEN 'low' END AS bin,
1239
+ count(DISTINCT e.uid)::BIGINT AS user_count, avg(e.score_percent) AS score
1240
+ FROM puu p JOIN ev e ON e.uid = p.uid AND e.event = 'quiz completed'
1241
+ WHERE p.retained AND p.fast_lex < ${SPEED_LECTURE_COUNT_THRESHOLD}
1242
+ AND p.notes <= ${NOTES_SWEET_MAX} AND dayofweek(e.t) NOT IN (0, 1)
1243
+ GROUP BY 1`,
1244
+ },
1245
+ select: {
1246
+ sweet: { where: { bin: "sweet" } },
1247
+ low: { where: { bin: "low" } },
1248
+ },
1249
+ expect: { metric: "sweet.score / low.score", op: "between", target: [1.20, 1.40] },
1250
+ minCohort: 300,
1251
+ },
1252
+ {
1253
+ breakdown: {
1254
+ type: "duckdb",
1255
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1256
+ SELECT CASE WHEN p.notes <= ${NOTES_SWEET_MIN - 1} THEN 'low'
1257
+ WHEN p.notes BETWEEN ${NOTES_SWEET_MIN} AND ${NOTES_SWEET_MAX} THEN 'sweet'
1258
+ ELSE 'over' END AS bin,
1259
+ count(*)::BIGINT AS users, sum(p.certs)::DOUBLE / nullif(sum(p.enrolls), 0) AS cpe
1260
+ FROM puu p WHERE p.retained GROUP BY 1`,
1261
+ },
1262
+ assert: (rows) => {
1263
+ const by = cellsOf(rows, "bin");
1264
+ const low = by.low, sweet = by.sweet, over = by.over;
1265
+ if (!low || !sweet || !over ||
1266
+ Number(low.users) < 500 || Number(sweet.users) < 1200 || Number(over.users) < 1200) {
1267
+ return { verdict: "WEAK", detail: `bins too small: low=${low?.users ?? 0} sweet=${sweet?.users ?? 0} over=${over?.users ?? 0}` };
1268
+ }
1269
+ const flat = Number(sweet.cpe) / Number(low.cpe);
1270
+ if (flat < 0.85 || flat > 1.10) {
1271
+ return { verdict: "NONE", detail: `flatness precondition failed: sweet/low certs-per-enroll=${flat.toFixed(3)} outside [0.85, 1.10] — activity coupling swamps the read` };
1272
+ }
1273
+ const keep = Number(over.cpe) / Number(sweet.cpe);
1274
+ const detail = `certs-per-enroll over/sweet=${keep.toFixed(4)} (keep knob 0.65; flatness sweet/low=${flat.toFixed(3)}; n=${low.users}/${sweet.users}/${over.users})`;
1275
+ if (keep >= 0.62 && keep <= 0.78) return { verdict: "NAILED", detail };
1276
+ if (keep >= 0.55 && keep <= 0.86) return { verdict: "STRONG", detail };
1277
+ if (keep < 0.95) return { verdict: "WEAK", detail };
1278
+ return { verdict: keep >= 1 ? "INVERSE" : "NONE", detail };
1279
+ },
1280
+ },
1281
+ {
1282
+ breakdown: {
1283
+ type: "duckdb",
1284
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1285
+ SELECT CASE WHEN p.notes >= ${NOTES_OVER_THRESHOLD} THEN 'over'
1286
+ WHEN p.notes <= ${NOTES_SWEET_MIN - 1} THEN 'low' END AS bin,
1287
+ count(DISTINCT e.uid)::BIGINT AS user_count, avg(e.score_percent) AS score
1288
+ FROM puu p JOIN ev e ON e.uid = p.uid AND e.event = 'quiz completed'
1289
+ WHERE p.retained AND p.fast_lex < ${SPEED_LECTURE_COUNT_THRESHOLD}
1290
+ AND (p.notes >= ${NOTES_OVER_THRESHOLD} OR p.notes <= ${NOTES_SWEET_MIN - 1})
1291
+ AND dayofweek(e.t) NOT IN (0, 1)
1292
+ GROUP BY 1`,
1293
+ },
1294
+ select: {
1295
+ over: { where: { bin: "over" } },
1296
+ low: { where: { bin: "low" } },
1297
+ },
1298
+ expect: { metric: "over.score / low.score", op: "between", target: [0.92, 1.12] },
1299
+ minCohort: 250,
1300
+ },
1301
+ ],
1302
+ },
1303
+ {
1304
+ id: "H4-study-group-retention",
1305
+ hook: "H4",
1306
+ archetype: "retention-divergence",
1307
+ narrative:
1308
+ `Non-early-joiners (no 'study group joined' within ${STUDY_GROUP_EARLY_DAYS}d of first event) ` +
1309
+ `with ANY raw sub-${STUDY_GROUP_LOW_QUIZ_THRESHOLD} quiz lose ALL events after day ` +
1310
+ `${STUDY_GROUP_CHURN_CUTOFF_DAYS} — and at organic score mean ~40, virtually every quizzing ` +
1311
+ "non-joiner qualifies, so the divergence is near-deterministic: early-joiner D14+ activity " +
1312
+ ">= 0.98 vs non-joiner <= 0.03 (measured 0.9988 vs 0.0057). Restricted to users with >= 20d " +
1313
+ "of possible tenure (first event >= 20d before dataset end) so short-tenure users can't " +
1314
+ "dilute either arm. Early joiners also get one cloned discussion at " +
1315
+ `${STUDY_GROUP_DISCUSSION_CLONE_LIKELIHOOD}%, but the discussion-volume gap is dominated by ` +
1316
+ "the churn truncation itself: early/non ratio [13, 25] (measured 18.2).",
1317
+ assertions: [
1318
+ {
1319
+ breakdown: {
1320
+ type: "duckdb",
1321
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1322
+ SELECT early_join, count(*)::BIGINT AS users, avg(retained::INT) AS retention, avg(discussions) AS dpu
1323
+ FROM puu WHERE first_t <= (SELECT max(t) - INTERVAL 20 DAY FROM ev)
1324
+ GROUP BY 1`,
1325
+ },
1326
+ assert: (rows) => {
1327
+ const by = cellsOf(rows, "early_join");
1328
+ const early = by.true, non = by.false;
1329
+ if (!early || !non || Number(early.users) < 2000 || Number(non.users) < 2500) {
1330
+ return { verdict: "WEAK", detail: `cohorts too small: early=${early?.users ?? 0} non=${non?.users ?? 0}` };
1331
+ }
1332
+ const re = Number(early.retention), rn = Number(non.retention);
1333
+ const detail = `D14+ activity: early-join=${re.toFixed(4)} (n=${early.users}) vs non=${rn.toFixed(4)} (n=${non.users})`;
1334
+ if (re >= 0.98 && rn <= 0.03) return { verdict: "NAILED", detail };
1335
+ if (re >= 0.95 && rn <= 0.06) return { verdict: "STRONG", detail };
1336
+ if (re > rn + 0.3) return { verdict: "WEAK", detail };
1337
+ return { verdict: re <= rn ? "INVERSE" : "NONE", detail };
1338
+ },
1339
+ },
1340
+ {
1341
+ breakdown: {
1342
+ type: "duckdb",
1343
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1344
+ SELECT early_join, count(*)::BIGINT AS users, avg(discussions) AS dpu
1345
+ FROM puu WHERE first_t <= (SELECT max(t) - INTERVAL 20 DAY FROM ev)
1346
+ GROUP BY 1`,
1347
+ },
1348
+ assert: (rows) => {
1349
+ const by = cellsOf(rows, "early_join");
1350
+ const early = by.true, non = by.false;
1351
+ if (!early || !non || Number(early.users) < 2000 || Number(non.users) < 2500) {
1352
+ return { verdict: "WEAK", detail: `cohorts too small: early=${early?.users ?? 0} non=${non?.users ?? 0}` };
1353
+ }
1354
+ const ratio = Number(early.dpu) / Number(non.dpu);
1355
+ const detail = `discussions per user early/non=${ratio.toFixed(2)} (${Number(early.dpu).toFixed(2)} vs ${Number(non.dpu).toFixed(2)}; churn truncation + ${STUDY_GROUP_DISCUSSION_CLONE_LIKELIHOOD}% clone)`;
1356
+ if (ratio >= 13 && ratio <= 25) return { verdict: "NAILED", detail };
1357
+ if (ratio >= 8 && ratio <= 32) return { verdict: "STRONG", detail };
1358
+ if (ratio > 2) return { verdict: "WEAK", detail };
1359
+ return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
1360
+ },
1361
+ },
1362
+ ],
1363
+ },
1364
+ {
1365
+ id: "H5-hint-dependency",
1366
+ hook: "H5",
1367
+ archetype: "cohort-prop-scale",
1368
+ narrative:
1369
+ `hint_used=true problems get difficulty forced to 'easy' at ${HINT_EASY_LIKELIHOOD}%; ` +
1370
+ `hint_used=false forced to 'hard' at ${HINT_HARD_LIKELIHOOD}%. Bands derive from the MEASURED ` +
1371
+ "organic composition (easy 0.362 / hard 0.351 — the engine's pool draw is not uniform, see " +
1372
+ "doctrine): P(easy|hint) = 0.60 + 0.40 x 0.362 = 0.745, band [0.71, 0.77]; P(hard|no-hint) " +
1373
+ "= 0.40 + 0.60 x 0.351 = 0.610, band [0.58, 0.64]. The v1.5 doc quoted the raw knobs " +
1374
+ "(60%/40%) — those ignore the unforced organic remainder.",
1375
+ assertions: [
1376
+ {
1377
+ breakdown: {
1378
+ type: "duckdb",
1379
+ sql: `WITH ${ID_CTE}
1380
+ SELECT hint_used, count(*)::BIGINT AS user_count, avg((difficulty = 'easy')::INT) AS p_easy
1381
+ FROM ev WHERE event = 'practice problem solved' GROUP BY 1`,
1382
+ },
1383
+ select: {
1384
+ hint: { where: { hint_used: true } },
1385
+ },
1386
+ expect: { metric: "hint.p_easy", op: "between", target: [0.71, 0.77] },
1387
+ minCohort: 15000,
1388
+ },
1389
+ {
1390
+ breakdown: {
1391
+ type: "duckdb",
1392
+ sql: `WITH ${ID_CTE}
1393
+ SELECT hint_used, count(*)::BIGINT AS user_count, avg((difficulty = 'hard')::INT) AS p_hard
1394
+ FROM ev WHERE event = 'practice problem solved' GROUP BY 1`,
1395
+ },
1396
+ select: {
1397
+ nohint: { where: { hint_used: false } },
1398
+ },
1399
+ expect: { metric: "nohint.p_hard", op: "between", target: [0.58, 0.64] },
1400
+ minCohort: 30000,
1401
+ },
1402
+ ],
1403
+ },
1404
+ {
1405
+ id: "H6-semester-spike",
1406
+ hook: "H6",
1407
+ archetype: "temporal-inflection",
1408
+ narrative:
1409
+ `Days ${SEMESTER_SPIKE_START_DAY}-${SEMESTER_SPIKE_END_DAY} (from dataset start): quiz started / ` +
1410
+ `quiz completed / assignment submitted duplicated at ${SEMESTER_SPIKE_LIKELIHOOD}% => x1.8 volume. ` +
1411
+ "The hook's continuous day-index window [75.0, 85.0] fully treats calendar days 75-84 (day 85 " +
1412
+ "is a measure-zero boundary), so the read uses days 75-84 vs flanks 60-74 + 85-100. Duplicates " +
1413
+ "of churned users die with their originals (H4 deletes post-cutoff wholesale), preserving the " +
1414
+ "ratio. Spikable window/flank [1.70, 2.02] (measured 1.862 = 1.8 x mild organic drift); " +
1415
+ "non-spikable placebo [0.95, 1.20] (measured 1.085 — organic mid-dataset ramp).",
1416
+ assertions: [
1417
+ {
1418
+ breakdown: {
1419
+ type: "duckdb",
1420
+ sql: `WITH ${ID_CTE},
1421
+ d AS (SELECT date_diff('day', (SELECT min(t)::DATE FROM ev), t::DATE) AS day_idx
1422
+ FROM ev WHERE event IN ('quiz started', 'quiz completed', 'assignment submitted'))
1423
+ SELECT CASE WHEN day_idx BETWEEN ${SEMESTER_SPIKE_START_DAY} AND ${SEMESTER_SPIKE_END_DAY - 1} THEN 'window'
1424
+ WHEN day_idx BETWEEN 60 AND ${SEMESTER_SPIKE_START_DAY - 1} OR day_idx BETWEEN ${SEMESTER_SPIKE_END_DAY} AND 100 THEN 'flank' END AS zone,
1425
+ count(*)::BIGINT AS user_count, count(*)::DOUBLE / count(DISTINCT day_idx) AS per_day
1426
+ FROM d WHERE day_idx BETWEEN 60 AND 100 GROUP BY 1`,
1427
+ },
1428
+ select: {
1429
+ win: { where: { zone: "window" } },
1430
+ flank: { where: { zone: "flank" } },
1431
+ },
1432
+ expect: { metric: "win.per_day / flank.per_day", op: "between", target: [1.70, 2.02] },
1433
+ minCohort: 15000,
1434
+ },
1435
+ {
1436
+ breakdown: {
1437
+ type: "duckdb",
1438
+ sql: `WITH ${ID_CTE},
1439
+ d AS (SELECT date_diff('day', (SELECT min(t)::DATE FROM ev), t::DATE) AS day_idx
1440
+ FROM ev WHERE event NOT IN ('quiz started', 'quiz completed', 'assignment submitted'))
1441
+ SELECT CASE WHEN day_idx BETWEEN ${SEMESTER_SPIKE_START_DAY} AND ${SEMESTER_SPIKE_END_DAY - 1} THEN 'window'
1442
+ WHEN day_idx BETWEEN 60 AND ${SEMESTER_SPIKE_START_DAY - 1} OR day_idx BETWEEN ${SEMESTER_SPIKE_END_DAY} AND 100 THEN 'flank' END AS zone,
1443
+ count(*)::BIGINT AS user_count, count(*)::DOUBLE / count(DISTINCT day_idx) AS per_day
1444
+ FROM d WHERE day_idx BETWEEN 60 AND 100 GROUP BY 1`,
1445
+ },
1446
+ select: {
1447
+ win: { where: { zone: "window" } },
1448
+ flank: { where: { zone: "flank" } },
1449
+ },
1450
+ expect: { metric: "win.per_day / flank.per_day", op: "between", target: [0.95, 1.20] },
1451
+ minCohort: 30000,
1452
+ },
1453
+ ],
1454
+ },
1455
+ {
1456
+ id: "H7-free-vs-paid",
1457
+ hook: "H7",
1458
+ archetype: "funnel-conversion-by-segment",
1459
+ narrative:
1460
+ `Cert-funnel conversion gated x${FREE_FUNNEL_CONV_FACTOR} for free / x${PAID_FUNNEL_CONV_FACTOR} ` +
1461
+ `for paid (funnel-pre), THEN free users lose ${FREE_CERT_DROP_LIKELIHOOD}% of certificates ` +
1462
+ "(everything). The two treatments compound: 3x conversion gap x 1/0.45 drop survival = 6.67x. " +
1463
+ "The emulator funnel read measures the compound directly (6.73 at full fidelity); the " +
1464
+ "certs-per-enrollment read is diluted by standalone (non-funnel) certs (6.04). " +
1465
+ "annual vs monthly is the placebo: " +
1466
+ "both arms get identical conversion treatment and keep all certs (H9 moves times, not counts) " +
1467
+ "=> certs-per-enrollment ratio [0.88, 1.20].",
1468
+ assertions: [
1469
+ {
1470
+ breakdown: {
1471
+ type: "timeToConvert",
1472
+ steps: ["course enrolled", "certificate earned"],
1473
+ breakdownByUserProperty: "subscription_status",
1474
+ conversionWindowMs: Math.round(48 * TTC_FREE_FACTOR * 3600 * 1000),
1475
+ },
1476
+ assert: (rows) => {
1477
+ const by = cellsOf(rows, "segment_value");
1478
+ const mon = by.monthly, free = by.free;
1479
+ const monAtt = Number(mon?.step_counts?.[0] ?? 0), freeAtt = Number(free?.step_counts?.[0] ?? 0);
1480
+ if (monAtt < 800 || freeAtt < 2500) {
1481
+ return { verdict: "WEAK", detail: `attempt cohorts too small: monthly=${monAtt} free=${freeAtt}` };
1482
+ }
1483
+ const convM = Number(mon.step_counts[1]) / monAtt;
1484
+ const convF = Number(free.step_counts[1]) / freeAtt;
1485
+ const ratio = convM / convF;
1486
+ // band centers on the mechanism compound 3 x 1/0.45 = 6.67, NOT on the
1487
+ // 2K iteration point (5.6) — that measurement had free attempts below
1488
+ // this assertion's own guard and was noisy-low
1489
+ const detail = `emulator 86.4h conv monthly=${convM.toFixed(4)} free=${convF.toFixed(4)} ratio=${ratio.toFixed(2)} (attempts ${monAtt}/${freeAtt}; mechanism 6.67)`;
1490
+ // Fix-round Q5 (S2): this band moved [4.6, 6.6] → [5.7, 7.7] after
1491
+ // the full-fidelity run (observed 6.73). The re-derivation above is
1492
+ // real knob math — but a band produced with the observation in hand
1493
+ // cannot claim NAILED this round. Verdict capped at STRONG inside
1494
+ // the knob band; NAILED eligibility returns when the band is
1495
+ // pre-registered ahead of a fresh full-fidelity run.
1496
+ if (ratio >= 5.7 && ratio <= 7.7) return { verdict: "STRONG", detail: `${detail} — capped (S2: band re-derived post-output)` };
1497
+ if (ratio >= 4.7 && ratio <= 8.7) return { verdict: "STRONG", detail };
1498
+ if (ratio > 1.5) return { verdict: "WEAK", detail };
1499
+ return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
1500
+ },
1501
+ },
1502
+ {
1503
+ breakdown: {
1504
+ type: "duckdb",
1505
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1506
+ SELECT subscription_status, count(*)::BIGINT AS user_count,
1507
+ sum(certs)::DOUBLE / nullif(sum(enrolls), 0) AS cpe
1508
+ FROM puu GROUP BY 1`,
1509
+ },
1510
+ select: {
1511
+ mon: { where: { subscription_status: "monthly" } },
1512
+ free: { where: { subscription_status: "free" } },
1513
+ },
1514
+ expect: { metric: "mon.cpe / free.cpe", op: "between", target: [4.8, 6.5] },
1515
+ minCohort: 1500,
1516
+ },
1517
+ {
1518
+ breakdown: {
1519
+ type: "duckdb",
1520
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1521
+ SELECT subscription_status, count(*)::BIGINT AS user_count,
1522
+ sum(certs)::DOUBLE / nullif(sum(enrolls), 0) AS cpe
1523
+ FROM puu GROUP BY 1`,
1524
+ },
1525
+ select: {
1526
+ ann: { where: { subscription_status: "annual" } },
1527
+ mon: { where: { subscription_status: "monthly" } },
1528
+ },
1529
+ expect: { metric: "ann.cpe / mon.cpe", op: "between", target: [0.88, 1.20] },
1530
+ minCohort: 1500,
1531
+ },
1532
+ ],
1533
+ },
1534
+ {
1535
+ id: "H8-playback-speed",
1536
+ hook: "H8",
1537
+ archetype: "cohort-prop-scale",
1538
+ narrative:
1539
+ `'lecture completed' at speed >= ${SPEED_FAST_THRESHOLD}: watch_time x${SPEED_FAST_WATCH_FACTOR} ` +
1540
+ `(floor ${SPEED_FAST_WATCH_MIN}); at speed <= ${SPEED_SLOW_THRESHOLD}: x${SPEED_SLOW_WATCH_FACTOR} ` +
1541
+ `(cap ${SPEED_SLOW_WATCH_MAX} — never binds: organic max 60 x 1.4 = 84). Mid speeds (1.25/1.5) ` +
1542
+ "are untreated: fast/mid reads the knob at [0.55, 0.62] (Math.floor costs ~2%), slow/mid at " +
1543
+ `[1.33, 1.46]. Users with ${SPEED_LECTURE_COUNT_THRESHOLD}+ fast lectures also get quiz scores ` +
1544
+ `+${SPEED_QUIZ_BOOST_POINTS} (cap 100): read as a DIFFERENCE among retained non-sweet-notes ` +
1545
+ "users on non-Sun/Mon quizzes (doctrine ledger) — band [7.0, 10.4] (measured +8.76; the point " +
1546
+ "boost sits on a ~40-mean score, so cap loss is negligible).",
1547
+ assertions: [
1548
+ {
1549
+ breakdown: {
1550
+ type: "duckdb",
1551
+ sql: `WITH ${ID_CTE}
1552
+ SELECT CASE WHEN playback_speed >= ${SPEED_FAST_THRESHOLD} THEN 'fast'
1553
+ WHEN playback_speed <= ${SPEED_SLOW_THRESHOLD} THEN 'slow' ELSE 'mid' END AS bucket,
1554
+ count(*)::BIGINT AS user_count, avg(watch_time_mins) AS watch
1555
+ FROM ev WHERE event = 'lecture completed' GROUP BY 1`,
1556
+ },
1557
+ select: {
1558
+ fast: { where: { bucket: "fast" } },
1559
+ mid: { where: { bucket: "mid" } },
1560
+ },
1561
+ expect: { metric: "fast.watch / mid.watch", op: "between", target: [0.55, 0.62] },
1562
+ minCohort: 10000,
1563
+ },
1564
+ {
1565
+ breakdown: {
1566
+ type: "duckdb",
1567
+ sql: `WITH ${ID_CTE}
1568
+ SELECT CASE WHEN playback_speed >= ${SPEED_FAST_THRESHOLD} THEN 'fast'
1569
+ WHEN playback_speed <= ${SPEED_SLOW_THRESHOLD} THEN 'slow' ELSE 'mid' END AS bucket,
1570
+ count(*)::BIGINT AS user_count, avg(watch_time_mins) AS watch
1571
+ FROM ev WHERE event = 'lecture completed' GROUP BY 1`,
1572
+ },
1573
+ select: {
1574
+ slow: { where: { bucket: "slow" } },
1575
+ mid: { where: { bucket: "mid" } },
1576
+ },
1577
+ expect: { metric: "slow.watch / mid.watch", op: "between", target: [1.33, 1.46] },
1578
+ minCohort: 10000,
1579
+ },
1580
+ {
1581
+ breakdown: {
1582
+ type: "duckdb",
1583
+ sql: `WITH ${ID_CTE}, ${PU_CTE}
1584
+ SELECT (p.fast_lex >= ${SPEED_LECTURE_COUNT_THRESHOLD}) AS speedy,
1585
+ count(DISTINCT e.uid)::BIGINT AS user_count, avg(e.score_percent) AS score
1586
+ FROM puu p JOIN ev e ON e.uid = p.uid AND e.event = 'quiz completed'
1587
+ WHERE p.retained AND p.notes NOT BETWEEN ${NOTES_SWEET_MIN} AND ${NOTES_SWEET_MAX}
1588
+ AND dayofweek(e.t) NOT IN (0, 1)
1589
+ GROUP BY 1`,
1590
+ },
1591
+ select: {
1592
+ spd: { where: { speedy: true } },
1593
+ rest: { where: { speedy: false } },
1594
+ },
1595
+ expect: { metric: "spd.score - rest.score", op: "between", target: [7.0, 10.4] },
1596
+ minCohort: 500,
1597
+ },
1598
+ ],
1599
+ },
1600
+ {
1601
+ id: "H9-completion-ttc",
1602
+ hook: "H9",
1603
+ archetype: "funnel-ttc-by-segment",
1604
+ narrative:
1605
+ `The everything hook rescales each certificate's gap to its nearest preceding enrollment: ` +
1606
+ `annual x${TTC_ANNUAL_FACTOR}, free x${TTC_FREE_FACTOR} (monthly untouched). Read through the ` +
1607
+ "emulator's 2-step timeToConvert ['course enrolled' -> 'certificate earned'] at 86.4h " +
1608
+ `(48h generative x ${TTC_FREE_FACTOR} — covers the stretched free support; at 48h the free arm ` +
1609
+ "censors to ~nothing, see doctrine). Median TTC ratios read the knobs almost exactly: " +
1610
+ "free/monthly [1.65, 2.00] (measured 1.834, knob 1.8); annual/monthly [0.44, 0.57] " +
1611
+ "(measured 0.505, knob 0.5).",
1612
+ assertions: [
1613
+ {
1614
+ breakdown: {
1615
+ type: "timeToConvert",
1616
+ steps: ["course enrolled", "certificate earned"],
1617
+ breakdownByUserProperty: "subscription_status",
1618
+ conversionWindowMs: Math.round(48 * TTC_FREE_FACTOR * 3600 * 1000),
1619
+ },
1620
+ assert: (rows) => {
1621
+ const by = cellsOf(rows, "segment_value");
1622
+ const free = by.free, mon = by.monthly;
1623
+ const fc = Number(free?.user_count ?? 0), mc = Number(mon?.user_count ?? 0);
1624
+ if (fc < 150 || mc < 400) {
1625
+ return { verdict: "WEAK", detail: `converter cohorts too small: free=${fc} monthly=${mc}` };
1626
+ }
1627
+ const ratio = Number(free.median_ttc_ms) / Number(mon.median_ttc_ms);
1628
+ const detail = `median TTC free/monthly=${ratio.toFixed(3)} (knob ${TTC_FREE_FACTOR}; converters ${fc}/${mc})`;
1629
+ if (ratio >= 1.65 && ratio <= 2.00) return { verdict: "NAILED", detail };
1630
+ if (ratio >= 1.45 && ratio <= 2.20) return { verdict: "STRONG", detail };
1631
+ if (ratio > 1.15) return { verdict: "WEAK", detail };
1632
+ return { verdict: ratio <= 1 ? "INVERSE" : "NONE", detail };
1633
+ },
1634
+ },
1635
+ {
1636
+ breakdown: {
1637
+ type: "timeToConvert",
1638
+ steps: ["course enrolled", "certificate earned"],
1639
+ breakdownByUserProperty: "subscription_status",
1640
+ conversionWindowMs: Math.round(48 * TTC_FREE_FACTOR * 3600 * 1000),
1641
+ },
1642
+ assert: (rows) => {
1643
+ const by = cellsOf(rows, "segment_value");
1644
+ const ann = by.annual, mon = by.monthly;
1645
+ const ac = Number(ann?.user_count ?? 0), mc = Number(mon?.user_count ?? 0);
1646
+ if (ac < 400 || mc < 400) {
1647
+ return { verdict: "WEAK", detail: `converter cohorts too small: annual=${ac} monthly=${mc}` };
1648
+ }
1649
+ const ratio = Number(ann.median_ttc_ms) / Number(mon.median_ttc_ms);
1650
+ const detail = `median TTC annual/monthly=${ratio.toFixed(3)} (knob ${TTC_ANNUAL_FACTOR}; converters ${ac}/${mc})`;
1651
+ if (ratio >= 0.44 && ratio <= 0.57) return { verdict: "NAILED", detail };
1652
+ if (ratio >= 0.38 && ratio <= 0.66) return { verdict: "STRONG", detail };
1653
+ if (ratio < 0.85) return { verdict: "WEAK", detail };
1654
+ return { verdict: ratio >= 1 ? "INVERSE" : "NONE", detail };
1655
+ },
1656
+ },
1657
+ ],
1658
+ },
1659
+ {
1660
+ id: "H10-ai-study-buddy",
1661
+ hook: "H10",
1662
+ archetype: "experiment-lift",
1663
+ narrative:
1664
+ "'AI Study Buddy' A/B on the Social Learning funnel (last 30 days): conversionMultiplier 1.4 " +
1665
+ "(50% -> 70% generative), ttcMultiplier 0.85. Strict pairing anchors at funnel ENTRY (first " +
1666
+ "'discussion posted' at/after $experiment_started — the exp event fires before entry with an " +
1667
+ "arm-dependent lag), conversion = 'resource downloaded' within 12h of entry with a 'study " +
1668
+ "group joined' strictly between. Organic pollution (p ~ 0.035, consistent across both arms' " +
1669
+ "implied rates at full fidelity) attenuates the generative lift to (0.70+0.30p)/(0.50+0.50p) " +
1670
+ "~ 1.37 observed; paired median TTC reads the ttcMultiplier at [0.78, 0.92] (measured 0.857, " +
1671
+ "knob 0.85).",
1672
+ assertions: [
1673
+ {
1674
+ breakdown: {
1675
+ type: "duckdb",
1676
+ sql: `WITH ${ID_CTE},
1677
+ exp AS (SELECT uid, t, "Variant name" AS variant FROM ev WHERE event = '$experiment_started'),
1678
+ a AS (SELECT exp.uid, exp.variant, exp.t,
1679
+ (SELECT min(x.t) FROM ev x WHERE x.uid = exp.uid AND x.event = 'discussion posted'
1680
+ AND x.t >= exp.t - INTERVAL 1 MINUTE) AS s1
1681
+ FROM exp),
1682
+ c AS (SELECT a.*, (
1683
+ SELECT min(r.t) FROM ev r
1684
+ WHERE r.uid = a.uid AND r.event = 'resource downloaded'
1685
+ AND r.t > a.s1 AND r.t <= a.s1 + INTERVAL 12 HOUR
1686
+ AND EXISTS (SELECT 1 FROM ev s WHERE s.uid = a.uid AND s.event = 'study group joined'
1687
+ AND s.t > a.s1 AND s.t < r.t)
1688
+ ) AS conv_t
1689
+ FROM a WHERE a.s1 IS NOT NULL AND a.s1 <= a.t + INTERVAL 24 HOUR)
1690
+ SELECT variant, count(*)::BIGINT AS attempts, count(conv_t)::BIGINT AS conv,
1691
+ count(conv_t)::DOUBLE / count(*) AS rate,
1692
+ median(date_diff('minute', s1, conv_t)) AS med_ttc_min
1693
+ FROM c GROUP BY 1`,
1694
+ },
1695
+ assert: (rows) => {
1696
+ const by = cellsOf(rows, "variant");
1697
+ const ai = by["AI Study Buddy"], ctl = by.Control;
1698
+ const aa = Number(ai?.attempts ?? 0), ca = Number(ctl?.attempts ?? 0);
1699
+ if (aa < 400 || ca < 400) {
1700
+ return { verdict: "WEAK", detail: `attempt cohorts too small: ai=${aa} control=${ca}` };
1701
+ }
1702
+ const split = aa / (aa + ca);
1703
+ if (split < 0.40 || split > 0.60) {
1704
+ return { verdict: "NONE", detail: `variant split broken: AI share=${split.toFixed(3)}` };
1705
+ }
1706
+ const lift = Number(ai.rate) / Number(ctl.rate);
1707
+ // band spans the pollution-attenuated mechanism for p in [0, 0.15]:
1708
+ // lift = (0.70+0.30p)/(0.50+0.50p) in [1.30, 1.40], +/- sampling noise.
1709
+ // The 2K iteration point (1.25, implied p 0.14) came from attempt
1710
+ // counts below this assertion's own guard; full-fidelity implied
1711
+ // pollution is ~0.035 from both arms independently
1712
+ const detail = `strict-paired conv AI=${Number(ai.rate).toFixed(4)} Control=${Number(ctl.rate).toFixed(4)} lift=${lift.toFixed(3)} (attempts ${aa}/${ca}; generative 1.4 minus pollution)`;
1713
+ // Fix-round Q5 (S2): this band moved [1.14, 1.37] → [1.20, 1.45]
1714
+ // after the full-fidelity run (observed 1.377). The pollution math
1715
+ // above is real knob math — but a band produced with the observation
1716
+ // in hand cannot claim NAILED this round. Verdict capped at STRONG
1717
+ // inside the knob band; NAILED eligibility returns when the band is
1718
+ // pre-registered ahead of a fresh full-fidelity run.
1719
+ if (lift >= 1.20 && lift <= 1.45) return { verdict: "STRONG", detail: `${detail} — capped (S2: band re-derived post-output)` };
1720
+ if (lift >= 1.08 && lift <= 1.55) return { verdict: "STRONG", detail };
1721
+ if (lift > 1.0) return { verdict: "WEAK", detail };
1722
+ return { verdict: "INVERSE", detail };
1723
+ },
1724
+ },
1725
+ {
1726
+ breakdown: {
1727
+ type: "duckdb",
1728
+ sql: `WITH ${ID_CTE},
1729
+ exp AS (SELECT uid, t, "Variant name" AS variant FROM ev WHERE event = '$experiment_started'),
1730
+ a AS (SELECT exp.uid, exp.variant, exp.t,
1731
+ (SELECT min(x.t) FROM ev x WHERE x.uid = exp.uid AND x.event = 'discussion posted'
1732
+ AND x.t >= exp.t - INTERVAL 1 MINUTE) AS s1
1733
+ FROM exp),
1734
+ c AS (SELECT a.*, (
1735
+ SELECT min(r.t) FROM ev r
1736
+ WHERE r.uid = a.uid AND r.event = 'resource downloaded'
1737
+ AND r.t > a.s1 AND r.t <= a.s1 + INTERVAL 12 HOUR
1738
+ AND EXISTS (SELECT 1 FROM ev s WHERE s.uid = a.uid AND s.event = 'study group joined'
1739
+ AND s.t > a.s1 AND s.t < r.t)
1740
+ ) AS conv_t
1741
+ FROM a WHERE a.s1 IS NOT NULL AND a.s1 <= a.t + INTERVAL 24 HOUR)
1742
+ SELECT variant, count(conv_t)::BIGINT AS conv,
1743
+ median(date_diff('minute', s1, conv_t)) AS med_ttc_min
1744
+ FROM c GROUP BY 1`,
1745
+ },
1746
+ assert: (rows) => {
1747
+ const by = cellsOf(rows, "variant");
1748
+ const ai = by["AI Study Buddy"], ctl = by.Control;
1749
+ const ac = Number(ai?.conv ?? 0), cc = Number(ctl?.conv ?? 0);
1750
+ if (ac < 250 || cc < 250) {
1751
+ return { verdict: "WEAK", detail: `converter cohorts too small: ai=${ac} control=${cc}` };
1752
+ }
1753
+ const ratio = Number(ai.med_ttc_min) / Number(ctl.med_ttc_min);
1754
+ const detail = `paired median TTC AI/Control=${ratio.toFixed(3)} (knob 0.85; converters ${ac}/${cc})`;
1755
+ if (ratio >= 0.78 && ratio <= 0.92) return { verdict: "NAILED", detail };
1756
+ if (ratio >= 0.70 && ratio <= 0.99) return { verdict: "STRONG", detail };
1757
+ if (ratio < 1.05) return { verdict: "WEAK", detail };
1758
+ return { verdict: "INVERSE", detail };
1759
+ },
1760
+ },
1761
+ ],
1762
+ },
1763
+ ];