@ak--47/dungeon-master 1.5.4 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +21 -12
  2. package/.claude/skills/create-dungeon/SKILL.md +65 -5
  3. package/.claude/skills/create-project/SKILL.md +1 -1
  4. package/.claude/skills/create-project/provision.mjs +57 -10
  5. package/.claude/skills/powertools/SKILL.md +75 -0
  6. package/.claude/skills/powertools/pt.mjs +91 -0
  7. package/.claude/skills/powertools/snapshot-project.mjs +124 -0
  8. package/.claude/skills/verify-dungeon/SKILL.md +56 -25
  9. package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
  10. package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
  11. package/.claude/skills/write-hooks/SKILL.md +156 -18
  12. package/CHANGELOG.md +224 -0
  13. package/HOOKS.md +566 -112
  14. package/README.md +30 -9
  15. package/docs/guides/1.3.0-upgrade-guide.md +262 -0
  16. package/docs/guides/1.3.1-upgrade-guide.md +328 -0
  17. package/docs/guides/1.4.0-upgrade-guide.md +130 -0
  18. package/docs/guides/1.4.1-upgrade-guide.md +125 -0
  19. package/docs/guides/1.4.3-upgrade-guide.md +200 -0
  20. package/docs/guides/1.4.4-upgrade-guide.md +95 -0
  21. package/docs/guides/1.5.0-upgrade-guide.md +426 -0
  22. package/docs/guides/1.5.1-upgrade-guide.md +178 -0
  23. package/docs/guides/1.5.3-upgrade-guide.md +117 -0
  24. package/docs/guides/1.6.0-upgrade-guide.md +204 -0
  25. package/dungeons/technical/stories-verify.js +152 -0
  26. package/dungeons/vertical/README.md +154 -0
  27. package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
  28. package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
  29. package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
  30. package/dungeons/vertical/community/community.js +1589 -0
  31. package/dungeons/vertical/community/community.sql +216 -0
  32. package/dungeons/vertical/community/community.verify.mjs +72 -0
  33. package/dungeons/vertical/crypto/crypto.js +1483 -0
  34. package/dungeons/vertical/crypto/crypto.sql +194 -0
  35. package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
  36. package/dungeons/vertical/dating/dating.js +1540 -0
  37. package/dungeons/vertical/dating/dating.sql +288 -0
  38. package/dungeons/vertical/dating/dating.verify.mjs +72 -0
  39. package/dungeons/vertical/devtools/devtools.js +1524 -0
  40. package/dungeons/vertical/devtools/devtools.sql +161 -0
  41. package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
  42. package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
  43. package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
  44. package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
  45. package/dungeons/vertical/education/education.js +1763 -0
  46. package/dungeons/vertical/education/education.sql +243 -0
  47. package/dungeons/vertical/education/education.verify.mjs +75 -0
  48. package/dungeons/vertical/fintech/fintech.js +1410 -0
  49. package/dungeons/vertical/fintech/fintech.sql +175 -0
  50. package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
  51. package/dungeons/vertical/fitness/fitness.js +1495 -0
  52. package/dungeons/vertical/fitness/fitness.sql +206 -0
  53. package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
  54. package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
  55. package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
  56. package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
  57. package/dungeons/vertical/gaming/gaming.js +1741 -0
  58. package/dungeons/vertical/gaming/gaming.sql +249 -0
  59. package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
  60. package/dungeons/vertical/healthcare/healthcare.js +1488 -0
  61. package/dungeons/vertical/healthcare/healthcare.sql +193 -0
  62. package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
  63. package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
  64. package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
  65. package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
  66. package/dungeons/vertical/logistics/logistics.js +1541 -0
  67. package/dungeons/vertical/logistics/logistics.sql +205 -0
  68. package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
  69. package/dungeons/vertical/marketplace/marketplace.js +1438 -0
  70. package/dungeons/vertical/marketplace/marketplace.sql +148 -0
  71. package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
  72. package/dungeons/vertical/{media.js → media/media.js} +557 -58
  73. package/dungeons/vertical/media/media.sql +173 -0
  74. package/dungeons/vertical/media/media.verify.mjs +74 -0
  75. package/dungeons/vertical/real-estate/real-estate.js +1629 -0
  76. package/dungeons/vertical/real-estate/real-estate.sql +167 -0
  77. package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
  78. package/dungeons/vertical/sass/sass.js +1733 -0
  79. package/dungeons/vertical/sass/sass.sql +173 -0
  80. package/dungeons/vertical/sass/sass.verify.mjs +74 -0
  81. package/dungeons/vertical/social/social.js +1456 -0
  82. package/dungeons/vertical/social/social.sql +160 -0
  83. package/dungeons/vertical/social/social.verify.mjs +76 -0
  84. package/dungeons/vertical/streaming/streaming.js +677 -0
  85. package/dungeons/vertical/streaming/streaming.sql +101 -0
  86. package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
  87. package/dungeons/vertical/support-desk/support-desk.js +741 -0
  88. package/dungeons/vertical/support-desk/support-desk.sql +137 -0
  89. package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
  90. package/dungeons/vertical/travel/travel.js +1603 -0
  91. package/dungeons/vertical/travel/travel.sql +236 -0
  92. package/dungeons/vertical/travel/travel.verify.mjs +72 -0
  93. package/index.js +7 -3
  94. package/lib/core/config-validator.js +37 -2
  95. package/lib/core/context.js +4 -2
  96. package/lib/core/dungeon-loader.js +20 -15
  97. package/lib/core/dungeon-to-json.js +22 -4
  98. package/lib/core/storage.js +3 -3
  99. package/lib/generators/funnels.js +36 -9
  100. package/lib/hook-helpers/cohort.js +53 -0
  101. package/lib/hook-helpers/index.js +8 -0
  102. package/lib/hook-helpers/mutate.js +1 -1
  103. package/lib/hook-helpers/shape.js +319 -0
  104. package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
  105. package/lib/hook-patterns/attributed-by-source.js +60 -58
  106. package/lib/hook-patterns/frequency-by-frequency.js +14 -4
  107. package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
  108. package/lib/hook-patterns/index.js +1 -1
  109. package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
  110. package/lib/orchestrators/user-loop.js +41 -0
  111. package/lib/templates/story-spec.schema.json +175 -0
  112. package/lib/utils/json-evaluator.js +1 -1
  113. package/lib/utils/utils.js +151 -59
  114. package/lib/verify/coerce.js +186 -0
  115. package/lib/verify/counting.js +270 -32
  116. package/lib/verify/emulate-breakdown.js +1285 -203
  117. package/lib/verify/first-time.js +89 -0
  118. package/lib/verify/flows.js +679 -0
  119. package/lib/verify/formula.js +259 -0
  120. package/lib/verify/funnel-engine.js +697 -113
  121. package/lib/verify/index.js +32 -4
  122. package/lib/verify/sessionize.js +275 -0
  123. package/lib/verify/story-runner.js +483 -0
  124. package/lib/verify/verify-dungeon.js +59 -37
  125. package/package.json +2 -1
  126. package/scripts/extract-dungeon-schema.mjs +2 -2
  127. package/scripts/run-many.mjs +2 -2
  128. package/scripts/verify-runner.mjs +2 -2
  129. package/scripts/verify-stories.mjs +244 -0
  130. package/types.d.ts +298 -30
  131. package/dungeons/vertical/ai-platform.js +0 -818
  132. package/dungeons/vertical/community.js +0 -966
  133. package/dungeons/vertical/crypto.js +0 -895
  134. package/dungeons/vertical/dating.js +0 -830
  135. package/dungeons/vertical/devtools.js +0 -1050
  136. package/dungeons/vertical/ecommerce.js +0 -811
  137. package/dungeons/vertical/education.js +0 -964
  138. package/dungeons/vertical/fintech.js +0 -946
  139. package/dungeons/vertical/fitness.js +0 -935
  140. package/dungeons/vertical/food-delivery.js +0 -934
  141. package/dungeons/vertical/gaming.js +0 -1120
  142. package/dungeons/vertical/healthcare.js +0 -945
  143. package/dungeons/vertical/insurance-application.js +0 -1039
  144. package/dungeons/vertical/logistics.js +0 -986
  145. package/dungeons/vertical/marketplace.js +0 -1019
  146. package/dungeons/vertical/real-estate.js +0 -872
  147. package/dungeons/vertical/sass.js +0 -995
  148. package/dungeons/vertical/social.js +0 -904
  149. package/dungeons/vertical/travel.js +0 -930
@@ -0,0 +1,1424 @@
1
+ // ── IMPORTS ──
2
+ import dayjs from "dayjs";
3
+ import utc from "dayjs/plugin/utc.js";
4
+ dayjs.extend(utc);
5
+ import "dotenv/config";
6
+ import * as u from "@ak--47/dungeon-master/utils";
7
+ /** @typedef {import("../../../types").Dungeon} Config */
8
+
9
+ // ── OVERVIEW ──
10
+ /*
11
+ * NAME: PromptForge
12
+ * APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
13
+ * requests for chat completions, embeddings, evaluations, and tool
14
+ * use. Billing is per input/output token. Features: prompt caching,
15
+ * tool use, multi-turn conversations, batch API, model selection,
16
+ * eval pipelines.
17
+ * SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
18
+ * CORE LOOP: organization created → api key created → api call → iterate
19
+ *
20
+ * EVENTS (18):
21
+ * api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
22
+ * > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
23
+ * > model selected (3) > api key created (2) > batch job submitted (2)
24
+ * > batch job completed (2) > billing payment (2) > member invited (2)
25
+ * > organization created (1) > api key rotated (1) > webhook configured (1)
26
+ * > account deactivated (1)
27
+ *
28
+ * FUNNELS (3):
29
+ * - Onboarding: organization created → api key created → api call (70%)
30
+ * - API to Eval Pipeline: api call → tool use call → eval job (45%)
31
+ * - Usage to Billing: api call → billing payment (30%)
32
+ *
33
+ * USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
34
+ * SUPER PROPS: api_tier, primary_use_case, sdk_language
35
+ * SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
36
+ * GROUPS: none
37
+ */
38
+
39
+ // ── HOOK STORIES ──
40
+ /*
41
+ * ---------------------------------------------------------------
42
+ * 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
43
+ * ---------------------------------------------------------------
44
+ *
45
+ * PATTERN: Customers who enable prompt caching see 70% lower
46
+ * cost_per_call. Once any api call has cache_enabled=true, all
47
+ * subsequent calls for that user get cost_usd reduced by 70%.
48
+ *
49
+ * HOW TO FIND IT IN MIXPANEL:
50
+ *
51
+ * Report 1: Cost Per Call by Cache Status
52
+ * - Report type: Insights
53
+ * - Event: "api call"
54
+ * - Measure: Average of "cost_usd"
55
+ * - Breakdown: "cache_enabled"
56
+ * - Expected: cache_enabled=true ~ $0.003, false ~ $0.01 (70% cheaper)
57
+ *
58
+ * Report 2: Cache Adoption Over Time
59
+ * - Report type: Insights
60
+ * - Event: "api call"
61
+ * - Measure: Total
62
+ * - Filter: cache_enabled = true
63
+ * - Line chart by week
64
+ * - Expected: steady growth in cached calls over the dataset
65
+ *
66
+ * REAL-WORLD ANALOGUE: Prompt caching avoids re-processing long
67
+ * system prompts on every call, dramatically reducing cost and latency.
68
+ *
69
+ * ---------------------------------------------------------------
70
+ * 2. MODEL MIGRATION WAVE (TIMED RELEASE — event)
71
+ * ---------------------------------------------------------------
72
+ *
73
+ * PATTERN: At day 60, new model "opus-4-7" releases. After day 60,
74
+ * 35% of api calls from Build/Enterprise users switch model to
75
+ * "opus-4-7". These calls use 1.5x tokens (smarter model, longer
76
+ * responses). "opus-4-7" is part of the DECLARED model enum
77
+ * (schema-first: hooks only write declared values); the hook rewrites
78
+ * engine-sampled opus-4-7 back to the pre-release mix, so the model
79
+ * exists ONLY after day 60 and ONLY on Build/Enterprise api calls —
80
+ * exact purity, asserted in the story.
81
+ *
82
+ * HOW TO FIND IT IN MIXPANEL:
83
+ *
84
+ * Report 1: Model Distribution Over Time
85
+ * - Report type: Insights
86
+ * - Event: "api call"
87
+ * - Measure: Total
88
+ * - Breakdown: "model"
89
+ * - Line chart by week
90
+ * - Expected: opus-4-7 appears at day 60, ramps to ~35% of paid calls
91
+ *
92
+ * Report 2: Tokens Per Model
93
+ * - Report type: Insights
94
+ * - Event: "api call"
95
+ * - Measure: Average of "tokens_used"
96
+ * - Breakdown: "model"
97
+ * - Expected: opus-4-7 ~ 1.5x tokens vs other models. Clean on
98
+ * non-agentic/non-batch users — Hooks 3 and 7 multiply tokens_used
99
+ * on their cohorts and blur the comparison if left in.
100
+ *
101
+ * REAL-WORLD ANALOGUE: New flagship model launches cause migration
102
+ * waves among power users who want improved capabilities.
103
+ *
104
+ * ---------------------------------------------------------------
105
+ * 3. AGENTIC LOOP POWER USERS (everything)
106
+ * ---------------------------------------------------------------
107
+ *
108
+ * PATTERN: Users with 3+ "tool use call" AND 3+ api-call events with
109
+ * multi_turn=true — counted on the post-churn stream (Hooks 4 and 8 run
110
+ * first, so the cohort is exactly rebuildable from the output) — get 8x
111
+ * tokens_used on api calls plus 2 extra cloned api-call events per
112
+ * existing (3x volume). Clones carry fresh insert_ids (Mixpanel dedupes
113
+ * on $insert_id — spread-cloning the template id would silently drop
114
+ * every clone at import), unique offset timestamps, and multi_turn=true,
115
+ * which pushes the cohort's multi-turn share to (0.25n + 2n)/3n ≈ 75%
116
+ * vs ~25% baseline — a verifier-visible signature of the 3x volume.
117
+ * COMPOUNDS with Hook 7 (deliberate): agentic ∩ batch users get
118
+ * 8x × 2x = 16x tokens_used — agentic batch workloads are the
119
+ * platform's whales. No flag — discover via cohort builder.
120
+ *
121
+ * HOW TO FIND IT IN MIXPANEL:
122
+ *
123
+ * Report 1: Tokens per User — Agentic Cohort
124
+ * - Report type: Insights (with cohort)
125
+ * - Cohort A: users with >= 3 "tool use call" AND >= 3 api-call with multi_turn=true
126
+ * - Cohort B: rest
127
+ * - Event: "api call"
128
+ * - Measure: Average of "tokens_used"
129
+ * - Expected: A ~ 8x B (16x for the batch overlap — exclude users
130
+ * with a "batch job submitted" from both cohorts for the clean 8x)
131
+ *
132
+ * REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
133
+ * tokens via extended tool-use loops.
134
+ *
135
+ * ---------------------------------------------------------------
136
+ * 4. RATE LIMIT CHURN (everything)
137
+ * ---------------------------------------------------------------
138
+ *
139
+ * PATTERN: Users with >= 2 "rate limit error" events in first 7 days
140
+ * churn at 60%: a churned user's ENTIRE post-week-1 stream is dropped
141
+ * (retention cliff), the surviving 40% keep everything. Per-user, not
142
+ * per-event thinning — thinning is unverifiable on a burst-selected
143
+ * cohort (any pre/post ratio inherits the selection week's decay; any
144
+ * cross-user ratio inherits activity selection; measured RoR landed at
145
+ * 0.17 vs the 0.4 knob even stratified on week-1 activity). The cliff
146
+ * gives a selection-free proportion instead: share of flagged users
147
+ * with ZERO post-week-1 events ≈ 0.60 (minus a tiny natural-quiet
148
+ * baseline, which the story cancels by differencing against the
149
+ * unflagged share). Classification basis is pre-week-1 and survives
150
+ * every drop in the file, so the cohort is exactly rebuildable from
151
+ * output. No flag — discover via cohort.
152
+ *
153
+ * HOW TO FIND IT IN MIXPANEL:
154
+ *
155
+ * Report 1: Retention by Early Rate-Limit Cohort
156
+ * - Report type: Retention
157
+ * - Cohort A: users with >= 2 "rate limit error" in first 7 days
158
+ * - Cohort B: rest
159
+ * - Expected: cohort A's retention collapses to ~40% of cohort B's
160
+ * from week 2 onward — a hard cliff, not a gradual decay. ~60% of
161
+ * cohort A never appears again after their first week.
162
+ *
163
+ * REAL-WORLD ANALOGUE: Developers who get rate-limited early often
164
+ * switch to a competitor — and when they go, they go completely.
165
+ *
166
+ * ---------------------------------------------------------------
167
+ * 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
168
+ * ---------------------------------------------------------------
169
+ *
170
+ * PATTERN: Free users have context_window=200000, Build=1000000,
171
+ * Enterprise=2000000. Enterprise users send 4x larger input_tokens.
172
+ * Context window and input tokens are scaled by tier.
173
+ *
174
+ * HOW TO FIND IT IN MIXPANEL:
175
+ *
176
+ * Report 1: Input Tokens by Tier
177
+ * - Report type: Insights
178
+ * - Event: "api call"
179
+ * - Measure: Average of "input_tokens"
180
+ * - Breakdown: "api_tier" (superProp)
181
+ * - Expected: Enterprise ~ 4x Free (Enterprise ~ 8K, Free ~ 2K)
182
+ *
183
+ * Report 2: Context Window by Tier
184
+ * - Report type: Insights
185
+ * - Event: "api call"
186
+ * - Measure: Average of "context_window"
187
+ * - Breakdown: "api_tier"
188
+ * - Expected: Free=200K, Build=1M, Enterprise=2M
189
+ *
190
+ * REAL-WORLD ANALOGUE: Enterprise customers pay for larger context
191
+ * windows and use them for long-document analysis and code review.
192
+ *
193
+ * ---------------------------------------------------------------
194
+ * 6. OUTAGE DAY (TIME-BASED — event)
195
+ * ---------------------------------------------------------------
196
+ *
197
+ * PATTERN: Days 40-41, is_error is set to true on 40% of api call
198
+ * events, error_type is set to a service error, latency_ms is tripled.
199
+ * Baseline api-call error rate is 0% BY SCHEMA (is_error declares
200
+ * [false]; "rate limit error" is a separate event) — the outage is the
201
+ * only source of api-call errors, so the window is exact: ~40% error
202
+ * share inside days 40-41, exactly zero outside.
203
+ *
204
+ * HOW TO FIND IT IN MIXPANEL:
205
+ *
206
+ * Report 1: Error Rate Over Time
207
+ * - Report type: Insights
208
+ * - Event: "api call"
209
+ * - Measure: Total
210
+ * - Filter: is_error = true
211
+ * - Line chart by day
212
+ * - Expected: two-day spike at days 40-41 (~40% of api calls),
213
+ * flat zero everywhere else
214
+ *
215
+ * Report 2: Error Types During Outage
216
+ * - Report type: Insights
217
+ * - Event: "api call"
218
+ * - Filter: is_error = true
219
+ * - Breakdown: "error_type"
220
+ * - Date range: days 40-41
221
+ * - Expected: service_overloaded and internal_server_error dominate
222
+ *
223
+ * REAL-WORLD ANALOGUE: API platforms experience periodic outages
224
+ * that spike error rates across all customers.
225
+ *
226
+ * ---------------------------------------------------------------
227
+ * 7. BATCH API DISCOUNT (everything)
228
+ * ---------------------------------------------------------------
229
+ *
230
+ * PATTERN: Users with any "batch job submitted" event (on the
231
+ * post-churn stream — Hooks 4 and 8 run first) get 50% lower
232
+ * cost_per_token on api calls + 2x tokens_used. Mutates raw props.
233
+ * cost_per_token is touched by NO other hook — clean 0.5x.
234
+ * tokens_used COMPOUNDS with Hook 3 (deliberate): agentic ∩ batch
235
+ * users get 2x × 8x = 16x — verified as its own cohort cell.
236
+ * No flag — discover via cohort builder.
237
+ *
238
+ * HOW TO FIND IT IN MIXPANEL:
239
+ *
240
+ * Report 1: Cost per Token by Batch Cohort
241
+ * - Report type: Insights (with cohort)
242
+ * - Cohort A: users with >= 1 "batch job submitted"
243
+ * - Cohort B: rest
244
+ * - Event: "api call"
245
+ * - Measure: Average of "cost_per_token"
246
+ * - Expected: A ~ 0.5x B
247
+ *
248
+ * REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
249
+ *
250
+ * ---------------------------------------------------------------
251
+ * 8. EVAL-DRIVEN RETENTION (everything)
252
+ * ---------------------------------------------------------------
253
+ *
254
+ * PATTERN: Users with any "eval job" in first 7 days keep all events.
255
+ * Non-eval users lose 75% of post-day-30 events (25% keep-rate). The
256
+ * classification basis is pre-week-1 and survives every drop in the
257
+ * file, so the cohort is exactly rebuildable from output. No flag —
258
+ * discover via retention cohort.
259
+ *
260
+ * HOW TO FIND IT IN MIXPANEL:
261
+ *
262
+ * Report 1: Retention by Early Eval Cohort
263
+ * - Report type: Retention
264
+ * - Cohort A: users with >= 1 "eval job" in first 7 days
265
+ * - Cohort B: rest
266
+ * - Expected: cohort B's post-day-30 event volume (relative to its
267
+ * own first-30-day volume) runs ~0.25x cohort A's — a hard drop in
268
+ * B's retention curve after day 30. The engineered constant is the
269
+ * 0.25 volume ratio-of-ratios, not a specific D30 percentage.
270
+ *
271
+ * REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
272
+ *
273
+ * ---------------------------------------------------------------
274
+ * 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
275
+ * ---------------------------------------------------------------
276
+ *
277
+ * PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
278
+ * 2x faster than baseline (factor 0.5 on inter-event gaps); Free users
279
+ * 2x slower (factor 2.0). Mutates funnel event timestamps.
280
+ *
281
+ * HOW TO FIND IT IN MIXPANEL:
282
+ *
283
+ * Report 1: API to Eval — Median Time-to-Convert by Tier
284
+ * - Report type: Funnels
285
+ * - Steps: "api call" -> "tool use call" -> "eval job"
286
+ * - Measure: Median time to convert
287
+ * - Breakdown: "api_tier"
288
+ * - Expected: Enterprise < Build < Free, Enterprise well under
289
+ * Build's median, Free well over. The measured ratios sit between
290
+ * the pure factors (0.5x / 2.0x) and 1: greedy first-match funnel
291
+ * evaluation (Mixpanel's and the emulator's) pairs organic events
292
+ * into instances, diluting toward 1 — and dilution is asymmetric
293
+ * (stretched Free gaps intercept more organic events than
294
+ * compressed Enterprise gaps). The factors are 0.5/2.0 precisely
295
+ * so the report-visible signal survives that dilution.
296
+ *
297
+ * NOTE (funnel-post measurement): visible via funnel median TTC
298
+ * (Mixpanel report or emulateBreakdown timeToConvert). Cross-event
299
+ * MIN→MIN SQL on raw events does NOT show this — funnel-post adjusts
300
+ * gaps within funnel instances, not across the user's full history.
301
+ *
302
+ * REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
303
+ * who execute end-to-end pipelines faster.
304
+ *
305
+ * ---------------------------------------------------------------
306
+ * 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
307
+ * ---------------------------------------------------------------
308
+ *
309
+ * PATTERN: Count "docs searched" events strictly between the EARLIEST
310
+ * organization-created and the EARLIEST billing-payment (by time, not
311
+ * array order). Sweet 1-2 → amount_usd × 1.35 on ALL billing payments.
312
+ * Over 3+ → amount_usd × 0.75 on ALL billing payments. Zero docs →
313
+ * untouched baseline. Only born-in-dataset users have an organization
314
+ * created event AND a billing payment (~4.6% of users), so thresholds
315
+ * are calibrated to the real docs_ct distribution in that segment
316
+ * (measured 0/1/2/3/4/5+ ≈ 43/19/14/12/10/2 per-cent — the old 5+
317
+ * "over" bin held ~7 users at full fidelity, a dead branch). Both
318
+ * effects are amount mutations, deliberately: amount_usd draws iid
319
+ * from the declared distribution regardless of user activity, so
320
+ * median ratios recover the knobs selection-free — count effects on
321
+ * ~100-user cohorts drown in activity-selection bias (a placebo test
322
+ * on the untouched 3+ cohort read 1.22-1.30 under the best count
323
+ * normalizer we found). No flag.
324
+ *
325
+ * HOW TO FIND IT IN MIXPANEL:
326
+ *
327
+ * Report 1: Median Billing Amount by Docs-Searched Bucket
328
+ * - Report type: Insights (with cohort)
329
+ * - Cohort A: users with 1-2 "docs searched" between org creation and
330
+ * first billing; Cohort B: users with 0; Cohort C: users with 3+
331
+ * - Event: "billing payment"
332
+ * - Measure: Median of "amount_usd"
333
+ * - Expected: A ~ 1.35x B; C ~ 0.75x B
334
+ *
335
+ * REAL-WORLD ANALOGUE: A little docs reading lifts willingness to pay;
336
+ * doc obsession signals a team stuck on integration that downgrades.
337
+ *
338
+ * ===============================================================
339
+ * EXPECTED METRICS SUMMARY
340
+ * ===============================================================
341
+ *
342
+ * Hook | Metric | Expected | Measured (full)
343
+ * ----------------------------|-------------------------------------------|---------------|----------------
344
+ * H1 Prompt Caching | avg cost_usd cached/uncached | 0.3x | 0.3005
345
+ * H1 Prompt Caching | share of users with any cached call | ~25% | 0.2297
346
+ * H2 Model Migration | opus-4-7 pre-day-60 / Free / non-api-call | 0 (exact) | 0 of 35975
347
+ * H2 Model Migration | opus-4-7 share, paid calls post-day-60 | ~35% | 0.3484
348
+ * H2 Model Migration | tokens opus/other (non-agentic/non-batch) | 1.5x | 1.461
349
+ * H3 Agentic Power Users | avg tokens agentic-only / neither | 8x | 7.875
350
+ * H3 Agentic Power Users | avg tokens agentic∩batch / neither | 16x (with H7) | 15.97
351
+ * H3 Agentic Power Users | multi-turn share of agentic api calls | ~75% (3x vol) | 0.7644
352
+ * H4 Rate Limit Churn | zero-post-week-1 share, flagged − rest | ~+0.60 diff | +0.6072 (flagged 0.6079)
353
+ * H5 Tier Context Window | avg input_tokens Enterprise / Free | 4x | 3.984
354
+ * H5 Tier Context Window | context_window per tier | 200K/1M/2M | exact (min=max per tier)
355
+ * H6 Outage Day | api-call error share, days 40-41 | ~40% (0% out) | 0.3996 (0 out-of-window)
356
+ * H7 Batch Discount | avg cost_per_token batch / rest | 0.5x | 0.4968
357
+ * H7 Batch Discount | avg tokens batch-only / neither | 2x | 2.159
358
+ * H8 Eval Retention | post/pre day-30 volume, noneval vs eval | 0.25x RoR | 0.2552
359
+ * H9 API-to-Eval TTC | funnel median TTC: Ent/Build, Free/Build | <1 / >1 (0.5, 2.0 pure; diluted) | 0.6606 / 1.263 (emulator, 336h window)
360
+ * H10 Docs Magic Number | median amount_usd sweet(1-2) / zero | 1.35x | 1.456
361
+ * H10 Docs Magic Number | median amount_usd over(3+) / zero | 0.75x | 0.8371
362
+ */
363
+
364
+ // ── SCALE ──
365
+ const SEED = "promptforge";
366
+ const NUM_USERS = 10_000;
367
+ const DATASET_START = "2026-01-01T00:00:00Z";
368
+ const DATASET_END = "2026-05-01T23:59:59Z";
369
+ const EVENTS_PER_DAY = 0.83;
370
+ const token = process.env.MP_TOKEN || "your-mixpanel-token";
371
+
372
+ const chance = u.initChance(SEED);
373
+
374
+ // ── KNOBS (tweak these to reshape stories) ──
375
+ const OUTAGE_START_DAY = 40;
376
+ const OUTAGE_END_DAY = 42;
377
+ const OUTAGE_ERROR_LIKELIHOOD = 40;
378
+ const OUTAGE_LATENCY_MULT = 3;
379
+
380
+ const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
381
+ const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
382
+
383
+ const CACHE_USER_HASH_MOD = 4;
384
+ const CACHE_COST_FACTOR = 0.3;
385
+ const CACHE_ACTIVATION_PCT = 0.2;
386
+
387
+ const MODEL_MIGRATION_DAY = 60;
388
+ const MODEL_MIGRATION_LIKELIHOOD = 35;
389
+ const MODEL_MIGRATION_TOKEN_MULT = 1.5;
390
+ // pre-release model mix — the declared api-call enum minus opus-4-7. H2
391
+ // rewrites engine-sampled opus-4-7 back to this mix so the model exists only
392
+ // after release day (schema-first requires opus-4-7 in the declared enum).
393
+ const PRE_RELEASE_MODELS = ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"];
394
+
395
+ const AGENTIC_TOOL_THRESHOLD = 3;
396
+ const AGENTIC_MULTITURN_THRESHOLD = 3;
397
+ const AGENTIC_TOKEN_MULT = 8;
398
+ const AGENTIC_CLONE_MULT = 2;
399
+
400
+ const RATE_LIMIT_THRESHOLD = 2;
401
+ // per-USER churn probability: a churned user loses ALL post-week-1 events
402
+ const RATE_LIMIT_CHURN_LIKELIHOOD = 60;
403
+
404
+ const BATCH_COST_FACTOR = 0.5;
405
+ const BATCH_TOKEN_MULT = 2;
406
+
407
+ const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
408
+ const EVAL_CUTOFF_DAYS = 30;
409
+
410
+ // bins calibrated to the measured docs_ct distribution among org∩billing
411
+ // users (~4.6% of users): 0 ≈ 43%, 1-2 ≈ 33%, 3+ ≈ 24% — every cohort
412
+ // clears ~100 users at full fidelity (the old 5+ bin held ~7: dead branch)
413
+ const DOCS_SWEET_MIN = 1;
414
+ const DOCS_SWEET_MAX = 2;
415
+ const DOCS_OVER_THRESHOLD = 3;
416
+ const DOCS_BILLING_BOOST = 1.35;
417
+ const DOCS_OVER_PENALTY = 0.75;
418
+
419
+ // 0.5/2.0 (not the 1.5 file's 0.67/1.4): greedy first-match funnel pairing
420
+ // dilutes measured TTC ratios toward 1 — these factors keep the report-
421
+ // visible signal clear of noise after dilution
422
+ const FUNNEL_TTC_ENTERPRISE = 0.5;
423
+ const FUNNEL_TTC_FREE = 2.0;
424
+
425
+ // ── HELPER FUNCTIONS ──
426
+ function handleFunnelPostHooks(record, meta) {
427
+ // H9: API-to-Eval TTC scaled by tier
428
+ const segment = meta?.profile?.api_tier;
429
+ if (Array.isArray(record) && record.length > 1) {
430
+ const factor = (
431
+ segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
432
+ segment === "Free" ? FUNNEL_TTC_FREE :
433
+ 1.0
434
+ );
435
+ if (factor !== 1.0) {
436
+ for (let i = 1; i < record.length; i++) {
437
+ const prev = dayjs(record[i - 1].time);
438
+ const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
439
+ record[i].time = prev.add(newGap, "milliseconds").toISOString();
440
+ }
441
+ }
442
+ }
443
+ return record;
444
+ }
445
+
446
+ function handleEverythingHooks(record, meta) {
447
+ const datasetStart = dayjs.unix(meta.datasetStart);
448
+ let events = record;
449
+ if (!events.length) return record;
450
+ const profile = meta && meta.profile ? meta.profile : {};
451
+
452
+ // ── ORDERING ──
453
+ // types.d.ts (HookMetaEverything) recommends stamp → mutate/clone → filter →
454
+ // temporal. This file deliberately diverges in ONE spot, documented here: the
455
+ // cohort-classified mutators (H3 agentic, H7 batch, H10 docs) run AFTER the
456
+ // filters (H4, H8) so each cohort's classification basis is exactly the
457
+ // surviving event stream. Classifying pre-drop makes the cohort unrecoverable
458
+ // from the output (the verifier cannot see dropped events) — the 1.5 file
459
+ // classified H3 before H4/H8's drops and its verifier "confirmed" 8x against
460
+ // a leaky cohort at 1.5x. None of H3/H7/H10 anchors on a dataset-day window,
461
+ // so post-filter classification costs nothing temporally. H6 (outage window)
462
+ // runs LAST per the types.d.ts rule so H3's clones landing inside days 40-41
463
+ // get error-stamped like every other call.
464
+
465
+ // ── PHASE 1: stamps ──
466
+ events.forEach(e => {
467
+ if (profile.api_tier) e.api_tier = profile.api_tier;
468
+ if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
469
+ if (profile.sdk_language) e.sdk_language = profile.sdk_language;
470
+ });
471
+
472
+ // H5: Tier-based context window & input tokens
473
+ const tier = profile.api_tier || "Free";
474
+ const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
475
+ const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
476
+ events.forEach(e => {
477
+ if (e.event === "api call") {
478
+ e.context_window = contextWindow;
479
+ e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
480
+ }
481
+ });
482
+
483
+ // H1: Prompt caching adoption — ~25% of users; activates ~20% into stream.
484
+ // Hash the PROFILE distinct_id, not events[0].user_id: the 1.5 file hashed
485
+ // events[0].user_id, which is undefined on device-only records —
486
+ // String(undefined || "") reduces to 0 and 0 % 4 === 0, silently classifying
487
+ // every such user as a cache user.
488
+ const hashBasis = String(profile.distinct_id || (events.find(e => e.user_id) || {}).user_id || "");
489
+ const idHash = hashBasis.split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
490
+ const isCacheUser = hashBasis.length > 0 && (idHash % CACHE_USER_HASH_MOD) === 0;
491
+ if (isCacheUser) {
492
+ let cacheActivated = false;
493
+ const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
494
+ events.forEach((e, idx) => {
495
+ if (e.event === "api call") {
496
+ if (idx >= activationPoint) cacheActivated = true;
497
+ if (cacheActivated) {
498
+ e.cache_enabled = true;
499
+ e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
500
+ }
501
+ }
502
+ });
503
+ }
504
+
505
+ // H2: Model migration wave.
506
+ // "opus-4-7" is in the DECLARED model enum (schema-first: hooks only write
507
+ // declared values), which means the engine samples it uniformly across the
508
+ // whole window — but the story needs zero opus-4-7 before release day.
509
+ // Rewrite every engine-sampled opus-4-7 back to the pre-release mix first;
510
+ // opus-4-7 in the output therefore comes from the migration stamp alone
511
+ // (post-day-60, Build/Enterprise api calls only — exact purity, asserted).
512
+ events.forEach(e => {
513
+ if (e.event === "api call" && e.model === "opus-4-7") {
514
+ e.model = chance.pickone(PRE_RELEASE_MODELS);
515
+ }
516
+ });
517
+ const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
518
+ if (tier === "Build" || tier === "Enterprise") {
519
+ events.forEach(e => {
520
+ if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
521
+ if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
522
+ e.model = "opus-4-7";
523
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
524
+ }
525
+ }
526
+ });
527
+ }
528
+
529
+ // First-event anchor for both churn filters. Computed pre-filter, but always
530
+ // verifier-recoverable: the filters only drop events STRICTLY after
531
+ // t0 + 7d / t0 + 30d, so min(time) in the output still equals t0.
532
+ const firstEventTime = events.reduce((min, e) => {
533
+ const t = dayjs(e.time);
534
+ return t.isBefore(min) ? t : min;
535
+ }, dayjs(events[0].time));
536
+ const firstWeekEnd = firstEventTime.add(7, "days");
537
+
538
+ // ── PHASE 2: filters ──
539
+ // H4: Rate-limit churn — 2+ early rate-limit errors → 60% of flagged users
540
+ // lose their ENTIRE post-week-1 stream. Per-USER cliff, not per-event
541
+ // thinning: thinning is unverifiable on a burst-selected cohort (pre/post
542
+ // ratios inherit the selection week's decay; cross-user ratios inherit
543
+ // activity selection), while the cliff yields a selection-free proportion —
544
+ // share of flagged users with zero post-week-1 events ≈ 0.6. Classification
545
+ // basis is pre-week-1 and survives every drop.
546
+ const earlyRateLimits = events.filter(e =>
547
+ e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
548
+ ).length;
549
+ if (earlyRateLimits >= RATE_LIMIT_THRESHOLD && chance.bool({ likelihood: RATE_LIMIT_CHURN_LIKELIHOOD })) {
550
+ events = events.filter(e => !dayjs(e.time).isAfter(firstWeekEnd));
551
+ }
552
+
553
+ // H8: Eval-driven retention — non-eval users lose 75% of post-day-30 events
554
+ // (classification basis is pre-week-1 and survives every drop)
555
+ const hasEarlyEval = events.some(e =>
556
+ e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
557
+ );
558
+ if (!hasEarlyEval) {
559
+ const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
560
+ events = events.filter(e => {
561
+ if (dayjs(e.time).isAfter(cutoff)) {
562
+ return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
563
+ }
564
+ return true;
565
+ });
566
+ }
567
+
568
+ // ── PHASE 3: cohort mutators + clones (classified on the SURVIVING stream) ──
569
+ // H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens,
570
+ // 2 clones per surviving api call (3x volume). Clones need FRESH insert_ids:
571
+ // the engine stamps insert_id at generation (lib/generators/events.js), so a
572
+ // bare spread copies the template's id and Mixpanel's $insert_id dedupe
573
+ // silently drops every clone at import — the 1.5 file shipped that bug.
574
+ // Clones stamp multi_turn: true, pushing the cohort's multi-turn share to
575
+ // (0.25n + 2n)/3n ≈ 75% — the verifier-visible signature of the 3x volume.
576
+ // Classification stays exactly recoverable from output: clones only ADD
577
+ // multi-turn api calls to users already at/above both thresholds.
578
+ // COMPOUND (deliberate): H7 below also multiplies tokens_used on these same
579
+ // events, clones included — agentic ∩ batch users land at 8 × 2 = 16x. The
580
+ // story verifies all four cells (neither/agentic/batch/both = 1x/8x/2x/16x).
581
+ const toolUseCount = events.filter(e => e.event === "tool use call").length;
582
+ const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
583
+ const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
584
+ if (isAgenticUser) {
585
+ events.forEach(e => {
586
+ if (e.event === "api call") {
587
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
588
+ }
589
+ });
590
+ const apiCalls = events.filter(e => e.event === "api call");
591
+ const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
592
+ for (let i = 0; i < extraCount; i++) {
593
+ const template = apiCalls[i % apiCalls.length];
594
+ if (template) {
595
+ events.push({
596
+ ...template,
597
+ insert_id: chance.guid(),
598
+ time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
599
+ user_id: template.user_id,
600
+ multi_turn: true,
601
+ });
602
+ }
603
+ }
604
+ }
605
+
606
+ // H7: Batch API discount — any surviving batch job submitted → 0.5x
607
+ // cost_per_token (touched by NO other hook — clean), 2x tokens_used
608
+ // (COMPOUNDS with H3, see above). Runs after H3 so the clones get the
609
+ // discount too — a batch user's api calls are uniformly discounted.
610
+ const isBatchUser = events.some(e => e.event === "batch job submitted");
611
+ if (isBatchUser) {
612
+ events.forEach(e => {
613
+ if (e.event === "api call") {
614
+ e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
615
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
616
+ }
617
+ });
618
+ }
619
+
620
+ // H10: Docs-searched magic number — docs strictly between the EARLIEST
621
+ // org-created and the EARLIEST billing payment (by time — the 1.5 file used
622
+ // Array.find, i.e. array order, on a not-yet-sorted stream). Both branches
623
+ // mutate amount_usd only (sweet ×1.35, over ×0.75): amounts draw iid from
624
+ // the declared distribution, so median ratios recover the knobs selection-
625
+ // free, and nothing is dropped — the classification window is always fully
626
+ // reconstructable from output.
627
+ const orgEvent = events.reduce((min, e) =>
628
+ e.event === "organization created" && (!min || dayjs(e.time).isBefore(dayjs(min.time))) ? e : min, null);
629
+ const firstBilling = events.reduce((min, e) =>
630
+ e.event === "billing payment" && (!min || dayjs(e.time).isBefore(dayjs(min.time))) ? e : min, null);
631
+ if (orgEvent && firstBilling) {
632
+ const aTime = dayjs(orgEvent.time);
633
+ const bTime = dayjs(firstBilling.time);
634
+ const docsBetween = events.filter(e =>
635
+ e.event === "docs searched" &&
636
+ dayjs(e.time).isAfter(aTime) &&
637
+ dayjs(e.time).isBefore(bTime)
638
+ ).length;
639
+ if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
640
+ events.forEach(e => {
641
+ if (e.event === "billing payment" && typeof e.amount_usd === "number") {
642
+ e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
643
+ }
644
+ });
645
+ } else if (docsBetween >= DOCS_OVER_THRESHOLD) {
646
+ events.forEach(e => {
647
+ if (e.event === "billing payment" && typeof e.amount_usd === "number") {
648
+ e.amount_usd = Math.round(e.amount_usd * DOCS_OVER_PENALTY);
649
+ }
650
+ });
651
+ }
652
+ }
653
+
654
+ // ── PHASE 4: temporal mutation LAST (clones in the window get stamped too) ──
655
+ // H6: Outage window [day 40, day 42) — 40% of api calls flagged is_error
656
+ // with a service error_type and 3x latency. Baseline is_error is 0% by
657
+ // schema (declared [false]), so the outage is the ONLY source of api-call
658
+ // errors: ~40% share inside the window, exactly zero outside.
659
+ events.forEach(e => {
660
+ if (e.event !== "api call") return;
661
+ const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
662
+ if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
663
+ if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
664
+ e.is_error = true;
665
+ e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
666
+ e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
667
+ }
668
+ }
669
+ });
670
+
671
+ return events;
672
+ }
673
+
674
+ // ── CONFIG ──
675
+ /** @type {Config} */
676
+ const config = {
677
+ version: 2,
678
+ seed: SEED,
679
+ datasetStart: DATASET_START,
680
+ datasetEnd: DATASET_END,
681
+ avgEventsPerUserPerDay: EVENTS_PER_DAY,
682
+ numUsers: NUM_USERS,
683
+ format: "json",
684
+ gzip: true,
685
+ credentials: {
686
+ token,
687
+ },
688
+ switches: {
689
+ hasSessionIds: true,
690
+ alsoInferFunnels: false,
691
+ hasLocation: true,
692
+ hasAndroidDevices: false,
693
+ hasIOSDevices: false,
694
+ hasDesktopDevices: true,
695
+ hasBrowser: true,
696
+ hasCampaigns: false,
697
+ isAnonymous: false,
698
+ hasAdSpend: false,
699
+ hasAvatar: true,
700
+ },
701
+ identity: {
702
+ avgDevicePerUser: 2,
703
+ },
704
+ concurrency: 1,
705
+ writeToDisk: false,
706
+ soup: "growth",
707
+ scdProps: {
708
+ monthly_api_usage: {
709
+ values: u.weighNumRange(0, 1000000, 0.3, 50),
710
+ frequency: "week",
711
+ timing: "fuzzy",
712
+ max: 20,
713
+ },
714
+ api_tier_history: {
715
+ values: ["Free", "Build", "Enterprise"],
716
+ frequency: "month",
717
+ timing: "fixed",
718
+ max: 6,
719
+ },
720
+ },
721
+ events: [
722
+ {
723
+ event: "organization created",
724
+ weight: 1,
725
+ isFirstEvent: true,
726
+ isAuthEvent: true,
727
+ properties: {
728
+ org_size: ["solo", "startup", "growth", "enterprise"],
729
+ referral_source: ["docs", "blog", "github", "word_of_mouth", "search", "conference"],
730
+ },
731
+ },
732
+ {
733
+ event: "api key created",
734
+ weight: 2,
735
+ properties: {
736
+ key_type: ["development", "production", "staging"],
737
+ key_scope: ["full_access", "read_only", "completions_only"],
738
+ },
739
+ },
740
+ {
741
+ event: "api key rotated",
742
+ weight: 1,
743
+ properties: {
744
+ rotation_reason: ["scheduled", "compromised", "policy", "manual"],
745
+ },
746
+ },
747
+ {
748
+ event: "api call",
749
+ weight: 10,
750
+ isStrictEvent: false,
751
+ properties: {
752
+ // opus-4-7 is declared so the H2 hook only writes declared values
753
+ // (schema-first); the hook scrubs engine-sampled opus-4-7 back to
754
+ // the pre-release mix, so it appears ONLY via the day-60 migration
755
+ model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6", "opus-4-7"],
756
+ input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
757
+ output_tokens: u.weighNumRange(10, 4000, 0.4, 500),
758
+ tokens_used: u.weighNumRange(100, 12000, 0.4, 2500),
759
+ cost_usd: [0.001, 0.002, 0.003, 0.003, 0.005, 0.005, 0.005, 0.008, 0.008, 0.01, 0.01, 0.01, 0.01, 0.015, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
760
+ cost_per_token: [0.000002, 0.000003, 0.000005, 0.000005, 0.000008, 0.000008, 0.00001, 0.00001, 0.00001, 0.000012, 0.000015, 0.00002, 0.000025, 0.00003],
761
+ latency_ms: u.weighNumRange(100, 15000, 0.4, 1500),
762
+ cache_enabled: [false],
763
+ is_error: [false],
764
+ error_type: ["none"],
765
+ multi_turn: [false, false, false, true],
766
+ context_window: [200000],
767
+ stream: [true, true, true, false],
768
+ stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
769
+ },
770
+ },
771
+ {
772
+ event: "tool use call",
773
+ weight: 4,
774
+ isStrictEvent: false,
775
+ properties: {
776
+ tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
777
+ execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
778
+ success: [true, true, true, true, false],
779
+ tool_input_tokens: u.weighNumRange(50, 2000, 0.4, 300),
780
+ tool_output_tokens: u.weighNumRange(20, 5000, 0.4, 500),
781
+ },
782
+ },
783
+ {
784
+ event: "batch job submitted",
785
+ weight: 2,
786
+ properties: {
787
+ batch_size: u.weighNumRange(10, 10000, 0.3, 500),
788
+ model: ["sonnet-4", "haiku-4", "opus-4-6"],
789
+ estimated_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
790
+ priority: ["standard", "standard", "standard", "express"],
791
+ },
792
+ },
793
+ {
794
+ event: "batch job completed",
795
+ weight: 2,
796
+ properties: {
797
+ batch_size: u.weighNumRange(10, 10000, 0.3, 500),
798
+ processing_time_sec: u.weighNumRange(60, 7200, 0.4, 900),
799
+ total_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
800
+ success_rate: u.weighNumRange(90, 100, 0.8, 98),
801
+ },
802
+ },
803
+ {
804
+ event: "eval job",
805
+ weight: 3,
806
+ isStrictEvent: false,
807
+ properties: {
808
+ eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
809
+ num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
810
+ model: ["sonnet-4", "haiku-4", "opus-4-6"],
811
+ dataset_name: ["prod_prompts", "safety_suite", "regression_set", "benchmark_v2", "custom_eval"],
812
+ },
813
+ },
814
+ {
815
+ event: "eval result",
816
+ weight: 3,
817
+ properties: {
818
+ eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
819
+ score: u.weighNumRange(0, 100, 0.6, 75),
820
+ pass_rate: u.weighNumRange(50, 100, 0.7, 85),
821
+ model: ["sonnet-4", "haiku-4", "opus-4-6"],
822
+ regression_detected: [false, false, false, false, true],
823
+ },
824
+ },
825
+ {
826
+ event: "rate limit error",
827
+ weight: 3,
828
+ properties: {
829
+ error_code: [429],
830
+ retry_after_ms: u.weighNumRange(1000, 60000, 0.3, 5000),
831
+ requests_per_minute: u.weighNumRange(50, 2000, 0.4, 500),
832
+ tier_limit: ["Free", "Build", "Enterprise"],
833
+ },
834
+ },
835
+ {
836
+ event: "billing payment",
837
+ weight: 2,
838
+ isStrictEvent: false,
839
+ properties: {
840
+ amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
841
+ payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
842
+ billing_period: ["monthly", "monthly", "annual"],
843
+ tokens_consumed: u.weighNumRange(100000, 50000000, 0.3, 5000000),
844
+ },
845
+ },
846
+ {
847
+ event: "model selected",
848
+ weight: 3,
849
+ properties: {
850
+ model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
851
+ is_default: [true, true, false],
852
+ selection_context: ["playground", "api_config", "eval_setup", "batch_config"],
853
+ },
854
+ },
855
+ {
856
+ event: "dashboard viewed",
857
+ weight: 5,
858
+ properties: {
859
+ dashboard_section: ["usage", "billing", "api_keys", "models", "evals", "logs"],
860
+ time_range: ["1h", "24h", "7d", "30d"],
861
+ },
862
+ },
863
+ {
864
+ event: "docs searched",
865
+ weight: 4,
866
+ properties: {
867
+ search_query_category: ["api_reference", "quickstart", "pricing", "models", "tool_use", "batch_api", "caching", "errors"],
868
+ results_found: u.weighNumRange(0, 50, 0.5, 8),
869
+ clicked_result: [true, true, true, false],
870
+ },
871
+ },
872
+ {
873
+ event: "member invited",
874
+ weight: 2,
875
+ properties: {
876
+ invite_role: ["admin", "developer", "developer", "billing", "viewer"],
877
+ invite_method: ["email", "email", "sso", "link"],
878
+ },
879
+ },
880
+ {
881
+ event: "webhook configured",
882
+ weight: 1,
883
+ properties: {
884
+ webhook_event: ["usage_alert", "rate_limit", "batch_complete", "eval_complete", "billing_threshold"],
885
+ delivery_method: ["https", "https", "slack", "email"],
886
+ },
887
+ },
888
+ {
889
+ event: "playground session",
890
+ weight: 4,
891
+ properties: {
892
+ model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
893
+ turns: u.weighNumRange(1, 30, 0.4, 5),
894
+ shared: [false, false, false, true],
895
+ tokens_used: u.weighNumRange(100, 20000, 0.3, 3000),
896
+ },
897
+ },
898
+ {
899
+ event: "account deactivated",
900
+ weight: 1,
901
+ isChurnEvent: true,
902
+ returnLikelihood: 0.1,
903
+ isStrictEvent: true,
904
+ properties: {
905
+ reason: ["cost", "switched_provider", "project_ended", "rate_limits", "no_longer_needed", "performance"],
906
+ },
907
+ },
908
+ ],
909
+ funnels: [
910
+ {
911
+ name: "Onboarding",
912
+ sequence: ["organization created", "api key created", "api call"],
913
+ conversionRate: 70,
914
+ order: "sequential",
915
+ isFirstFunnel: true,
916
+ timeToConvert: 48,
917
+ weight: 3,
918
+ },
919
+ {
920
+ name: "API to Eval Pipeline",
921
+ sequence: ["api call", "tool use call", "eval job"],
922
+ conversionRate: 45,
923
+ order: "sequential",
924
+ timeToConvert: 168,
925
+ weight: 5,
926
+ },
927
+ {
928
+ name: "Usage to Billing",
929
+ sequence: ["api call", "billing payment"],
930
+ conversionRate: 30,
931
+ order: "sequential",
932
+ timeToConvert: 336,
933
+ weight: 2,
934
+ },
935
+ ],
936
+ superProps: {
937
+ api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
938
+ primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
939
+ sdk_language: ["python", "typescript", "java", "go", "curl"],
940
+ },
941
+ userProps: {
942
+ api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
943
+ primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
944
+ sdk_language: ["python", "typescript", "java", "go", "curl"],
945
+ monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
946
+ total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
947
+ preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
948
+ },
949
+ hook(record, type, meta) {
950
+ if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
951
+ if (type === "everything") return handleEverythingHooks(record, meta);
952
+ return record;
953
+ },
954
+ };
955
+
956
+ // ── STORIES ──────────────────────────────────────────────────────────────
957
+ // Machine-checkable contract for the 10 numbered hooks. Evaluate with:
958
+ // node scripts/verify-stories.mjs dungeons/vertical/ai-platform/ai-platform.js --data-prefix verify-ai-platform
959
+
960
+ const EV = `read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true)`;
961
+ const US = `read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)`;
962
+
963
+ // Identity prelude. organization created is both isAuthEvent and isFirstEvent,
964
+ // so born users auth on their very first event and user_id should be present
965
+ // on every record; the prelude still resolves through the device pool
966
+ // (avgDevicePerUser: 2, "anonymousIds" is the legacy USERS-shard key) as
967
+ // belt-and-braces for any device-only edge.
968
+ const ID_CTE = `dmap AS (SELECT unnest("anonymousIds") AS device_id, distinct_id FROM ${US}),
969
+ ev AS (SELECT coalesce(m.distinct_id::VARCHAR, e.user_id::VARCHAR, e.device_id::VARCHAR) AS uid,
970
+ e.time::TIMESTAMP AS t, e.* FROM ${EV} e LEFT JOIN dmap m ON e.device_id = m.device_id)`;
971
+
972
+ // Temporal boundaries computed from the same knobs the hooks use
973
+ const MIG_TS = dayjs.utc(DATASET_START).add(MODEL_MIGRATION_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
974
+ const OUTAGE_START_TS = dayjs.utc(DATASET_START).add(OUTAGE_START_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
975
+ const OUTAGE_END_TS = dayjs.utc(DATASET_START).add(OUTAGE_END_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
976
+ // H4 eligibility: users must have ≥14d of post-week-1 runway, else a natural
977
+ // short tail is indistinguishable from the engineered cliff
978
+ const H4_ELIGIBLE_TS = dayjs.utc(DATASET_END).subtract(21, "day").format("YYYY-MM-DD HH:mm:ss");
979
+
980
+ // H3/H7 cohort cells — EXACTLY the hook's classification. The filters (H4/H8)
981
+ // run before the cohort mutators, so thresholds applied to the output
982
+ // reproduce the hook's cohorts 1:1 (H3's clones only add multi-turn api calls
983
+ // to users already at/above both agentic thresholds; non-members are untouched).
984
+ const CELL_CTE = `coh AS (SELECT e.uid,
985
+ (count(*) FILTER (WHERE e.event = 'tool use call') >= ${AGENTIC_TOOL_THRESHOLD}
986
+ AND count(*) FILTER (WHERE e.event = 'api call' AND e.multi_turn = true) >= ${AGENTIC_MULTITURN_THRESHOLD}) AS agentic,
987
+ bool_or(e.event = 'batch job submitted') AS batch
988
+ FROM ev e GROUP BY 1),
989
+ cells AS (SELECT uid, CASE WHEN agentic AND batch THEN 'both' WHEN agentic THEN 'agentic'
990
+ WHEN batch THEN 'batch' ELSE 'neither' END AS cell FROM coh)`;
991
+
992
+ // Per-user first-event anchor (H4/H8). min(time) in the output equals the
993
+ // hook's anchor because both filters only drop strictly-later events.
994
+ const T0_CTE = `t0 AS (SELECT uid, min(t) AS t0 FROM ev GROUP BY 1)`;
995
+
996
+ // H10 cohorts: docs strictly between earliest org-created and earliest billing
997
+ // payment. Both hook branches are amount-only mutations (nothing dropped, no
998
+ // events injected), so every event the hook classified on survives to the
999
+ // output — the window is exactly rebuildable. The else-bin is 'zero' (docs_ct
1000
+ // = 0, the modal case at ~43% of org∩billing users).
1001
+ const DOCS_CTE = `org AS (SELECT uid, min(t) AS org_t FROM ev WHERE event = 'organization created' GROUP BY 1),
1002
+ bill AS (SELECT uid, min(t) AS bill_t FROM ev WHERE event = 'billing payment' GROUP BY 1),
1003
+ docs AS (SELECT o.uid, count(e.uid) AS docs_ct
1004
+ FROM org o JOIN bill b ON b.uid = o.uid
1005
+ LEFT JOIN ev e ON e.uid = o.uid AND e.event = 'docs searched' AND e.t > o.org_t AND e.t < b.bill_t
1006
+ GROUP BY 1, o.org_t, b.bill_t),
1007
+ dcoh AS (SELECT uid, CASE WHEN docs_ct BETWEEN ${DOCS_SWEET_MIN} AND ${DOCS_SWEET_MAX} THEN 'sweet'
1008
+ WHEN docs_ct >= ${DOCS_OVER_THRESHOLD} THEN 'over' ELSE 'zero' END AS grp FROM docs)`;
1009
+
1010
+ /** @type {import("../../../types").DungeonStory[]} */
1011
+ export const stories = [
1012
+ {
1013
+ id: "H1-prompt-caching",
1014
+ hook: "H1",
1015
+ archetype: "cohort-prop-scale",
1016
+ narrative: `~25% of users (profile distinct_id charcode-sum % ${CACHE_USER_HASH_MOD} === 0) activate prompt caching ~${CACHE_ACTIVATION_PCT * 100}% into their stream; from then on api calls carry cache_enabled=true and cost_usd × ${CACHE_COST_FACTOR}. The flag is stamped, so the breakdown is direct`,
1017
+ assertions: [
1018
+ {
1019
+ breakdown: {
1020
+ type: "duckdb",
1021
+ sql: `SELECT CASE WHEN cache_enabled = true THEN 'cached' ELSE 'uncached' END AS grp,
1022
+ avg(cost_usd) AS avg_cost, count(*) AS event_count
1023
+ FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
1024
+ },
1025
+ select: { c: { where: { grp: "cached" } }, u: { where: { grp: "uncached" } } },
1026
+ // knob 0.3; both sides draw from the same declared cost distribution
1027
+ expect: { metric: "c.avg_cost / u.avg_cost", op: "between", target: [0.24, 0.36] },
1028
+ },
1029
+ {
1030
+ // hash cohort share: charcode-sum % 4 of GUID-ish ids ≈ uniform → ~25%
1031
+ breakdown: {
1032
+ type: "duckdb",
1033
+ sql: `WITH ${ID_CTE},
1034
+ per AS (SELECT uid, bool_or(cache_enabled = true) AS is_cache FROM ev WHERE event = 'api call' GROUP BY 1)
1035
+ SELECT 'all' AS grp, count(*) AS user_count,
1036
+ count(*) FILTER (WHERE is_cache)::DOUBLE / count(*) AS cache_share
1037
+ FROM per`,
1038
+ },
1039
+ select: { all: { where: { grp: "all" } } },
1040
+ expect: { metric: "all.cache_share", op: "between", target: [0.17, 0.33] },
1041
+ },
1042
+ ],
1043
+ },
1044
+ {
1045
+ id: "H2-model-migration",
1046
+ hook: "H2",
1047
+ archetype: "temporal-inflection",
1048
+ narrative: `opus-4-7 releases at day ${MODEL_MIGRATION_DAY}: ${MODEL_MIGRATION_LIKELIHOOD}% of post-release Build/Enterprise api calls migrate, at ${MODEL_MIGRATION_TOKEN_MULT}x tokens. The hook scrubs engine-sampled opus-4-7 back to the pre-release mix, so purity is exact: zero opus-4-7 before the release instant, on Free users, or on any non-api-call event`,
1049
+ assertions: [
1050
+ {
1051
+ // deterministic purity — the scrub + tier/date-gated stamp make
1052
+ // any impure row a hook bug, not sampling noise
1053
+ breakdown: {
1054
+ type: "duckdb",
1055
+ sql: `SELECT 'all' AS grp,
1056
+ count(*) FILTER (WHERE model = 'opus-4-7' AND (time::TIMESTAMP < TIMESTAMP '${MIG_TS}' OR api_tier = 'Free' OR event <> 'api call')) AS impure,
1057
+ count(*) FILTER (WHERE model = 'opus-4-7') AS opus_calls
1058
+ FROM ${EV} WHERE model IS NOT NULL`,
1059
+ },
1060
+ assert: (rows) => {
1061
+ const r = (rows || [])[0];
1062
+ if (!r) return { pass: false, verdict: "NONE", detail: "no rows" };
1063
+ if (Number(r.opus_calls) === 0) return { pass: false, verdict: "NONE", detail: "no opus-4-7 calls at all" };
1064
+ const clean = Number(r.impure) === 0;
1065
+ return {
1066
+ pass: clean,
1067
+ verdict: clean ? "NAILED" : "INVERSE",
1068
+ detail: `impure=${r.impure} of ${r.opus_calls} opus-4-7 rows (pre-release / Free / non-api-call must all be 0)`,
1069
+ };
1070
+ },
1071
+ },
1072
+ {
1073
+ // per-call migration is Bernoulli(0.35) — share of paid post-release calls
1074
+ breakdown: {
1075
+ type: "duckdb",
1076
+ sql: `SELECT 'all' AS grp, count(*) AS event_count,
1077
+ count(*) FILTER (WHERE model = 'opus-4-7')::DOUBLE / count(*) AS share
1078
+ FROM ${EV} WHERE event = 'api call' AND api_tier IN ('Build', 'Enterprise')
1079
+ AND time::TIMESTAMP >= TIMESTAMP '${MIG_TS}'`,
1080
+ },
1081
+ select: { all: { where: { grp: "all" } } },
1082
+ expect: { metric: "all.share", op: "between", target: [0.3, 0.4] },
1083
+ },
1084
+ {
1085
+ // tokens 1.5x — restricted to non-agentic/non-batch users so H3's 8x
1086
+ // and H7's 2x (which hit opus and non-opus calls of their cohorts
1087
+ // alike) can't blur the comparison
1088
+ breakdown: {
1089
+ type: "duckdb",
1090
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1091
+ SELECT CASE WHEN e.model = 'opus-4-7' THEN 'opus' ELSE 'other' END AS grp,
1092
+ avg(e.tokens_used) AS avg_tokens, count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1093
+ FROM ev e JOIN cells c ON c.uid = e.uid AND c.cell = 'neither'
1094
+ WHERE e.event = 'api call' AND e.api_tier IN ('Build', 'Enterprise')
1095
+ AND e.t >= TIMESTAMP '${MIG_TS}'
1096
+ GROUP BY 1`,
1097
+ },
1098
+ select: { o: { where: { grp: "opus" } }, x: { where: { grp: "other" } } },
1099
+ expect: { metric: "o.avg_tokens / x.avg_tokens", op: "between", target: [1.3, 1.7] },
1100
+ minCohort: 100,
1101
+ },
1102
+ ],
1103
+ },
1104
+ {
1105
+ id: "H3-agentic-power-users",
1106
+ hook: "H3",
1107
+ archetype: "cohort-prop-scale",
1108
+ narrative: `users with ${AGENTIC_TOOL_THRESHOLD}+ tool use calls AND ${AGENTIC_MULTITURN_THRESHOLD}+ multi-turn api calls (classified post-filter — exactly rebuildable) get ${AGENTIC_TOKEN_MULT}x tokens_used and ${AGENTIC_CLONE_MULT} clones per api call. Four-cell design with H7: neither/agentic/batch/both = 1x/${AGENTIC_TOKEN_MULT}x/${BATCH_TOKEN_MULT}x/${AGENTIC_TOKEN_MULT * BATCH_TOKEN_MULT}x. Clones stamp multi_turn=true → agentic multi-turn share ≈ 75% (the 3x-volume signature)`,
1109
+ assertions: [
1110
+ {
1111
+ breakdown: {
1112
+ type: "duckdb",
1113
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1114
+ SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
1115
+ count(*) FILTER (WHERE e.multi_turn = true)::DOUBLE / count(*) AS mt_share,
1116
+ count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1117
+ FROM cells c JOIN ev e ON e.uid = c.uid
1118
+ WHERE e.event = 'api call' GROUP BY 1`,
1119
+ },
1120
+ select: { a: { where: { grp: "agentic" } }, n: { where: { grp: "neither" } } },
1121
+ // knob 8x; H2's 1.5x rides both cells (tier ⊥ cohort) and cancels
1122
+ expect: { metric: "a.avg_tokens / n.avg_tokens", op: "between", target: [6.4, 9.6] },
1123
+ minCohort: 50,
1124
+ },
1125
+ {
1126
+ breakdown: {
1127
+ type: "duckdb",
1128
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1129
+ SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
1130
+ count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1131
+ FROM cells c JOIN ev e ON e.uid = c.uid
1132
+ WHERE e.event = 'api call' GROUP BY 1`,
1133
+ },
1134
+ select: { b: { where: { grp: "both" } }, n: { where: { grp: "neither" } } },
1135
+ // the deliberate H3×H7 compound: 8 × 2 = 16x
1136
+ expect: { metric: "b.avg_tokens / n.avg_tokens", op: "between", target: [12.8, 19.2] },
1137
+ minCohort: 40,
1138
+ },
1139
+ {
1140
+ breakdown: {
1141
+ type: "duckdb",
1142
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1143
+ SELECT c.cell AS grp,
1144
+ count(*) FILTER (WHERE e.multi_turn = true)::DOUBLE / count(*) AS mt_share,
1145
+ count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1146
+ FROM cells c JOIN ev e ON e.uid = c.uid
1147
+ WHERE e.event = 'api call' GROUP BY 1`,
1148
+ },
1149
+ select: { a: { where: { grp: "agentic" } } },
1150
+ // (0.25n + 2n)/3n = 0.75 — clone-volume signature; the declared
1151
+ // multi_turn mix is 1-in-4
1152
+ expect: { metric: "a.mt_share", op: "between", target: [0.62, 0.85] },
1153
+ minCohort: 50,
1154
+ },
1155
+ ],
1156
+ },
1157
+ {
1158
+ id: "H4-rate-limit-churn",
1159
+ hook: "H4",
1160
+ archetype: "retention-divergence",
1161
+ narrative: `${RATE_LIMIT_CHURN_LIKELIHOOD}% of users with ${RATE_LIMIT_THRESHOLD}+ rate limit errors in their first 7 days lose ALL post-week-1 events (per-user cliff). The signal is the share of flagged users with zero post-week-1 events, DIFFERENCED against the unflagged share to cancel the natural-quiet baseline — selection-free, unlike volume ratios on a burst-selected cohort. Restricted to users with ≥14d of post-week-1 runway`,
1162
+ assertions: [
1163
+ {
1164
+ breakdown: {
1165
+ type: "duckdb",
1166
+ sql: `WITH ${ID_CTE}, ${T0_CTE},
1167
+ rl AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
1168
+ WHERE e.event = 'rate limit error' AND e.t < t0.t0 + INTERVAL 7 DAY
1169
+ GROUP BY 1 HAVING count(*) >= ${RATE_LIMIT_THRESHOLD}),
1170
+ per AS (SELECT t0.uid, (t0.uid IN (SELECT uid FROM rl)) AS flagged,
1171
+ count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL 7 DAY) AS post_ct
1172
+ FROM t0 JOIN ev e ON e.uid = t0.uid
1173
+ WHERE t0.t0 <= TIMESTAMP '${H4_ELIGIBLE_TS}' GROUP BY 1, 2)
1174
+ SELECT CASE WHEN flagged THEN 'flagged' ELSE 'rest' END AS grp,
1175
+ count(*) AS user_count,
1176
+ count(*) FILTER (WHERE post_ct = 0)::DOUBLE / count(*) AS zero_share
1177
+ FROM per GROUP BY 1`,
1178
+ },
1179
+ select: { f: { where: { grp: "flagged" } }, r: { where: { grp: "rest" } } },
1180
+ // knob 0.6 churn probability; differencing cancels the baseline
1181
+ expect: { metric: "f.zero_share - r.zero_share", op: "between", target: [0.45, 0.7] },
1182
+ minCohort: 50,
1183
+ },
1184
+ {
1185
+ // direct knob readout: flagged zero-post share ≈ 0.6 + tiny baseline
1186
+ breakdown: {
1187
+ type: "duckdb",
1188
+ sql: `WITH ${ID_CTE}, ${T0_CTE},
1189
+ rl AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
1190
+ WHERE e.event = 'rate limit error' AND e.t < t0.t0 + INTERVAL 7 DAY
1191
+ GROUP BY 1 HAVING count(*) >= ${RATE_LIMIT_THRESHOLD}),
1192
+ per AS (SELECT t0.uid,
1193
+ count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL 7 DAY) AS post_ct
1194
+ FROM t0 JOIN ev e ON e.uid = t0.uid
1195
+ WHERE t0.uid IN (SELECT uid FROM rl) AND t0.t0 <= TIMESTAMP '${H4_ELIGIBLE_TS}'
1196
+ GROUP BY 1)
1197
+ SELECT 'flagged' AS grp, count(*) AS user_count,
1198
+ count(*) FILTER (WHERE post_ct = 0)::DOUBLE / count(*) AS zero_share
1199
+ FROM per`,
1200
+ },
1201
+ select: { f: { where: { grp: "flagged" } } },
1202
+ expect: { metric: "f.zero_share", op: "between", target: [0.5, 0.75] },
1203
+ minCohort: 50,
1204
+ },
1205
+ ],
1206
+ },
1207
+ {
1208
+ id: "H5-tier-context-window",
1209
+ hook: "H5",
1210
+ archetype: "cohort-prop-scale",
1211
+ narrative: `input_tokens scaled ${TIER_INPUT_MULT.Free}/${TIER_INPUT_MULT.Build}/${TIER_INPUT_MULT.Enterprise}x and context_window pinned to ${TIER_CONTEXT_WINDOW.Free}/${TIER_CONTEXT_WINDOW.Build}/${TIER_CONTEXT_WINDOW.Enterprise} by api_tier. No other hook touches either prop — clean stamp-phase constants`,
1212
+ assertions: [
1213
+ {
1214
+ breakdown: {
1215
+ type: "duckdb",
1216
+ sql: `SELECT api_tier AS grp, avg(input_tokens) AS avg_in, avg(context_window) AS avg_cw,
1217
+ count(*) AS event_count
1218
+ FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
1219
+ },
1220
+ select: { e: { where: { grp: "Enterprise" } }, f: { where: { grp: "Free" } } },
1221
+ // knob 4x (Math.floor truncation is sub-1% at these magnitudes)
1222
+ expect: { metric: "e.avg_in / f.avg_in", op: "between", target: [3.5, 4.5] },
1223
+ },
1224
+ {
1225
+ breakdown: {
1226
+ type: "duckdb",
1227
+ sql: `SELECT api_tier AS grp, avg(context_window) AS avg_cw, min(context_window) AS min_cw,
1228
+ max(context_window) AS max_cw, count(*) AS event_count
1229
+ FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
1230
+ },
1231
+ assert: (rows) => {
1232
+ const want = { Free: TIER_CONTEXT_WINDOW.Free, Build: TIER_CONTEXT_WINDOW.Build, Enterprise: TIER_CONTEXT_WINDOW.Enterprise };
1233
+ const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
1234
+ const bad = Object.entries(want).filter(([tier, cw]) =>
1235
+ !by[tier] || Number(by[tier].min_cw) !== cw || Number(by[tier].max_cw) !== cw);
1236
+ const detail = Object.keys(want).map((tr) => `${tr}=${by[tr] ? `${by[tr].min_cw}..${by[tr].max_cw}` : "missing"}`).join(" ");
1237
+ return {
1238
+ pass: bad.length === 0,
1239
+ verdict: bad.length === 0 ? "NAILED" : "INVERSE",
1240
+ detail: `${detail} (every api call must carry its tier's exact constant)`,
1241
+ };
1242
+ },
1243
+ },
1244
+ ],
1245
+ },
1246
+ {
1247
+ id: "H6-outage-day",
1248
+ hook: "H6",
1249
+ archetype: "temporal-inflection",
1250
+ narrative: `days ${OUTAGE_START_DAY}-${OUTAGE_END_DAY - 1}: ${OUTAGE_ERROR_LIKELIHOOD}% of api calls flagged is_error with a service error_type and ${OUTAGE_LATENCY_MULT}x latency. is_error declares [false], so the outage is the only error source — the window boundary is exact`,
1251
+ assertions: [
1252
+ {
1253
+ breakdown: {
1254
+ type: "duckdb",
1255
+ sql: `SELECT 'all' AS grp,
1256
+ count(*) FILTER (WHERE is_error = true AND time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}')::DOUBLE
1257
+ / nullif(count(*) FILTER (WHERE time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}'), 0) AS in_share,
1258
+ count(*) FILTER (WHERE time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}') AS in_calls,
1259
+ count(*) FILTER (WHERE is_error = true AND (time::TIMESTAMP < TIMESTAMP '${OUTAGE_START_TS}' OR time::TIMESTAMP >= TIMESTAMP '${OUTAGE_END_TS}')) AS out_errors
1260
+ FROM ${EV} WHERE event = 'api call'`,
1261
+ },
1262
+ select: { all: { where: { grp: "all" } } },
1263
+ // knob 40% (Bernoulli per in-window call)
1264
+ expect: { metric: "all.in_share", op: "between", target: [0.35, 0.45] },
1265
+ },
1266
+ {
1267
+ breakdown: {
1268
+ type: "duckdb",
1269
+ sql: `SELECT 'all' AS grp,
1270
+ count(*) FILTER (WHERE is_error = true AND (time::TIMESTAMP < TIMESTAMP '${OUTAGE_START_TS}' OR time::TIMESTAMP >= TIMESTAMP '${OUTAGE_END_TS}')) AS out_errors,
1271
+ count(*) FILTER (WHERE is_error = true) AS total_errors
1272
+ FROM ${EV} WHERE event = 'api call'`,
1273
+ },
1274
+ assert: (rows) => {
1275
+ const r = (rows || [])[0];
1276
+ if (!r) return { pass: false, verdict: "NONE", detail: "no rows" };
1277
+ if (Number(r.total_errors) === 0) return { pass: false, verdict: "NONE", detail: "no errors at all — outage never fired" };
1278
+ const clean = Number(r.out_errors) === 0;
1279
+ return {
1280
+ pass: clean,
1281
+ verdict: clean ? "NAILED" : "INVERSE",
1282
+ detail: `out-of-window errors=${r.out_errors} of ${r.total_errors} total (baseline is 0% by schema)`,
1283
+ };
1284
+ },
1285
+ },
1286
+ ],
1287
+ },
1288
+ {
1289
+ id: "H7-batch-discount",
1290
+ hook: "H7",
1291
+ archetype: "cohort-prop-scale",
1292
+ narrative: `users with any surviving batch job submitted get cost_per_token × ${BATCH_COST_FACTOR} (touched by no other hook) and tokens_used × ${BATCH_TOKEN_MULT} (compounds with H3 — the 'both' cell is verified in the H3 story)`,
1293
+ assertions: [
1294
+ {
1295
+ breakdown: {
1296
+ type: "duckdb",
1297
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1298
+ SELECT CASE WHEN c.cell IN ('batch', 'both') THEN 'batch' ELSE 'rest' END AS grp,
1299
+ avg(e.cost_per_token) AS avg_cpt, count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1300
+ FROM cells c JOIN ev e ON e.uid = c.uid
1301
+ WHERE e.event = 'api call' GROUP BY 1`,
1302
+ },
1303
+ select: { b: { where: { grp: "batch" } }, r: { where: { grp: "rest" } } },
1304
+ // knob 0.5; cost_per_token has no other mutator
1305
+ expect: { metric: "b.avg_cpt / r.avg_cpt", op: "between", target: [0.42, 0.58] },
1306
+ minCohort: 100,
1307
+ },
1308
+ {
1309
+ breakdown: {
1310
+ type: "duckdb",
1311
+ sql: `WITH ${ID_CTE}, ${CELL_CTE}
1312
+ SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
1313
+ count(*) AS event_count, count(DISTINCT e.uid) AS user_count
1314
+ FROM cells c JOIN ev e ON e.uid = c.uid
1315
+ WHERE e.event = 'api call' GROUP BY 1`,
1316
+ },
1317
+ select: { b: { where: { grp: "batch" } }, n: { where: { grp: "neither" } } },
1318
+ // batch-only cell: clean 2x (agentic users are in their own cells)
1319
+ expect: { metric: "b.avg_tokens / n.avg_tokens", op: "between", target: [1.7, 2.3] },
1320
+ minCohort: 100,
1321
+ },
1322
+ ],
1323
+ },
1324
+ {
1325
+ id: "H8-eval-retention",
1326
+ hook: "H8",
1327
+ archetype: "retention-divergence",
1328
+ narrative: `users without an eval job in their first 7 days keep only ${EVAL_NON_USER_KEEP_LIKELIHOOD}% of post-day-${EVAL_CUTOFF_DAYS} events. Ratio-of-ratios (noneval post/pre vs eval post/pre) cancels window lengths and the growth soup; H4's independent drop rides both cohorts`,
1329
+ assertions: [
1330
+ {
1331
+ breakdown: {
1332
+ type: "duckdb",
1333
+ sql: `WITH ${ID_CTE}, ${T0_CTE},
1334
+ ev_users AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
1335
+ WHERE e.event = 'eval job' AND e.t < t0.t0 + INTERVAL 7 DAY GROUP BY 1),
1336
+ per AS (SELECT t0.uid, (t0.uid IN (SELECT uid FROM ev_users)) AS eval_user,
1337
+ count(*) FILTER (WHERE e.t <= t0.t0 + INTERVAL ${EVAL_CUTOFF_DAYS} DAY) AS pre_ct,
1338
+ count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL ${EVAL_CUTOFF_DAYS} DAY) AS post_ct
1339
+ FROM t0 JOIN ev e ON e.uid = t0.uid GROUP BY 1, 2)
1340
+ SELECT CASE WHEN eval_user THEN 'eval' ELSE 'noneval' END AS grp,
1341
+ count(*) AS user_count, avg(post_ct) AS avg_post, avg(pre_ct) AS avg_pre,
1342
+ avg(post_ct) / nullif(avg(pre_ct), 0) AS post_pre
1343
+ FROM per GROUP BY 1`,
1344
+ },
1345
+ select: { n: { where: { grp: "noneval" } }, e: { where: { grp: "eval" } } },
1346
+ // knob keep-rate 0.25
1347
+ expect: { metric: "n.post_pre / e.post_pre", op: "between", target: [0.17, 0.34] },
1348
+ minCohort: 50,
1349
+ },
1350
+ ],
1351
+ },
1352
+ {
1353
+ id: "H9-api-to-eval-ttc",
1354
+ hook: "H9",
1355
+ archetype: "funnel-ttc-by-segment",
1356
+ narrative: `funnel-post scales API-to-Eval step gaps by tier: Enterprise × ${FUNNEL_TTC_ENTERPRISE}, Free × ${FUNNEL_TTC_FREE}, Build 1x. Measured with the Mixpanel-aligned funnel emulator (greedy step pairing), NOT raw SQL: nearest-preceding-pair SQL is censored by the fixed lookback window — stretching Free gaps pushes true pairs past the window edge and intercepts more organic events, which INVERTS the measured direction. The emulator window is 336h = max scale factor (${FUNNEL_TTC_FREE}) × the funnel's 168h generative window, so the stretched support fits — at 168h any Free instance whose original TTC exceeded 84h fails the window and the longest (most-stretched) pairs censor out. Greedy pairing still dilutes toward 1 (organic same-window events get picked as steps), asymmetrically — stretch (Free) dilutes harder than compress (Enterprise). Bands reflect the diluted effect, not the pure knobs`,
1357
+ assertions: [
1358
+ {
1359
+ breakdown: {
1360
+ type: "timeToConvert",
1361
+ steps: ["api call", "tool use call", "eval job"],
1362
+ breakdownByUserProperty: "api_tier",
1363
+ // 336h = FUNNEL_TTC_FREE × generative 168h window (covers stretched support)
1364
+ conversionWindowMs: 336 * 60 * 60 * 1000,
1365
+ },
1366
+ select: { e: { where: { segment_value: "Enterprise" } }, b: { where: { segment_value: "Build" } } },
1367
+ // knob 0.5 pure; greedy-pairing dilution pulls toward 1
1368
+ expect: { metric: "e.median_ttc_ms / b.median_ttc_ms", op: "between", target: [0.55, 0.92] },
1369
+ minCohort: 300,
1370
+ },
1371
+ {
1372
+ breakdown: {
1373
+ type: "timeToConvert",
1374
+ steps: ["api call", "tool use call", "eval job"],
1375
+ breakdownByUserProperty: "api_tier",
1376
+ // 336h = FUNNEL_TTC_FREE × generative 168h window (covers stretched support)
1377
+ conversionWindowMs: 336 * 60 * 60 * 1000,
1378
+ },
1379
+ select: { f: { where: { segment_value: "Free" } }, b: { where: { segment_value: "Build" } } },
1380
+ // knob 2.0 pure; stretch dilutes harder than compress
1381
+ expect: { metric: "f.median_ttc_ms / b.median_ttc_ms", op: "between", target: [1.1, 2.2] },
1382
+ minCohort: 300,
1383
+ },
1384
+ ],
1385
+ },
1386
+ {
1387
+ id: "H10-docs-magic-number",
1388
+ hook: "H10",
1389
+ archetype: "frequency-sweet-spot",
1390
+ narrative: `docs searched between org-created and first billing: ${DOCS_SWEET_MIN}-${DOCS_SWEET_MAX} (sweet) → amount_usd × ${DOCS_BILLING_BOOST} on all billing payments; ${DOCS_OVER_THRESHOLD}+ (over) → amount_usd × ${DOCS_OVER_PENALTY} on all billing payments. Both branches mutate an iid-drawn property — selection-free, unlike count effects which drown in activity-selection bias at ~100-user cohorts (placebo on an untouched cohort read 1.22-1.30 under the best normalizer). Median ratios against the untouched zero-docs cohort read the knobs directly (amount_usd draw is docs-count-independent; ×k is monotone so median scales by k; Math.round is sub-1% at these medians)`,
1391
+ assertions: [
1392
+ {
1393
+ breakdown: {
1394
+ type: "duckdb",
1395
+ sql: `WITH ${ID_CTE}, ${DOCS_CTE}
1396
+ SELECT d.grp AS grp, median(e.amount_usd) AS med_amt, count(*) AS payment_count,
1397
+ count(DISTINCT d.uid) AS user_count
1398
+ FROM dcoh d JOIN ev e ON e.uid = d.uid AND e.event = 'billing payment'
1399
+ GROUP BY 1`,
1400
+ },
1401
+ select: { s: { where: { grp: "sweet" } }, z: { where: { grp: "zero" } } },
1402
+ // knob 1.35 ±15%
1403
+ expect: { metric: "s.med_amt / z.med_amt", op: "between", target: [1.15, 1.55] },
1404
+ minCohort: 60,
1405
+ },
1406
+ {
1407
+ breakdown: {
1408
+ type: "duckdb",
1409
+ sql: `WITH ${ID_CTE}, ${DOCS_CTE}
1410
+ SELECT d.grp AS grp, median(e.amount_usd) AS med_amt, count(*) AS payment_count,
1411
+ count(DISTINCT d.uid) AS user_count
1412
+ FROM dcoh d JOIN ev e ON e.uid = d.uid AND e.event = 'billing payment'
1413
+ GROUP BY 1`,
1414
+ },
1415
+ select: { o: { where: { grp: "over" } }, z: { where: { grp: "zero" } } },
1416
+ // knob 0.75 ±15%
1417
+ expect: { metric: "o.med_amt / z.med_amt", op: "between", target: [0.64, 0.86] },
1418
+ minCohort: 60,
1419
+ },
1420
+ ],
1421
+ },
1422
+ ];
1423
+
1424
+ export default config;