@ak--47/dungeon-master 1.5.4 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +1 -1
- package/.claude/skills/create-project/provision.mjs +57 -10
- package/.claude/skills/powertools/SKILL.md +75 -0
- package/.claude/skills/powertools/pt.mjs +91 -0
- package/.claude/skills/powertools/snapshot-project.mjs +124 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +224 -0
- package/HOOKS.md +566 -112
- package/README.md +30 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +7 -3
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +151 -59
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +2 -1
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +298 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -0,0 +1,1424 @@
|
|
|
1
|
+
// ── IMPORTS ──
|
|
2
|
+
import dayjs from "dayjs";
|
|
3
|
+
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
5
|
+
import "dotenv/config";
|
|
6
|
+
import * as u from "@ak--47/dungeon-master/utils";
|
|
7
|
+
/** @typedef {import("../../../types").Dungeon} Config */
|
|
8
|
+
|
|
9
|
+
// ── OVERVIEW ──
|
|
10
|
+
/*
|
|
11
|
+
* NAME: PromptForge
|
|
12
|
+
* APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
|
|
13
|
+
* requests for chat completions, embeddings, evaluations, and tool
|
|
14
|
+
* use. Billing is per input/output token. Features: prompt caching,
|
|
15
|
+
* tool use, multi-turn conversations, batch API, model selection,
|
|
16
|
+
* eval pipelines.
|
|
17
|
+
* SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
|
|
18
|
+
* CORE LOOP: organization created → api key created → api call → iterate
|
|
19
|
+
*
|
|
20
|
+
* EVENTS (18):
|
|
21
|
+
* api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
|
|
22
|
+
* > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
|
|
23
|
+
* > model selected (3) > api key created (2) > batch job submitted (2)
|
|
24
|
+
* > batch job completed (2) > billing payment (2) > member invited (2)
|
|
25
|
+
* > organization created (1) > api key rotated (1) > webhook configured (1)
|
|
26
|
+
* > account deactivated (1)
|
|
27
|
+
*
|
|
28
|
+
* FUNNELS (3):
|
|
29
|
+
* - Onboarding: organization created → api key created → api call (70%)
|
|
30
|
+
* - API to Eval Pipeline: api call → tool use call → eval job (45%)
|
|
31
|
+
* - Usage to Billing: api call → billing payment (30%)
|
|
32
|
+
*
|
|
33
|
+
* USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
|
|
34
|
+
* SUPER PROPS: api_tier, primary_use_case, sdk_language
|
|
35
|
+
* SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
|
|
36
|
+
* GROUPS: none
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
// ── HOOK STORIES ──
|
|
40
|
+
/*
|
|
41
|
+
* ---------------------------------------------------------------
|
|
42
|
+
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
43
|
+
* ---------------------------------------------------------------
|
|
44
|
+
*
|
|
45
|
+
* PATTERN: Customers who enable prompt caching see 70% lower
|
|
46
|
+
* cost_per_call. Once any api call has cache_enabled=true, all
|
|
47
|
+
* subsequent calls for that user get cost_usd reduced by 70%.
|
|
48
|
+
*
|
|
49
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
50
|
+
*
|
|
51
|
+
* Report 1: Cost Per Call by Cache Status
|
|
52
|
+
* - Report type: Insights
|
|
53
|
+
* - Event: "api call"
|
|
54
|
+
* - Measure: Average of "cost_usd"
|
|
55
|
+
* - Breakdown: "cache_enabled"
|
|
56
|
+
* - Expected: cache_enabled=true ~ $0.003, false ~ $0.01 (70% cheaper)
|
|
57
|
+
*
|
|
58
|
+
* Report 2: Cache Adoption Over Time
|
|
59
|
+
* - Report type: Insights
|
|
60
|
+
* - Event: "api call"
|
|
61
|
+
* - Measure: Total
|
|
62
|
+
* - Filter: cache_enabled = true
|
|
63
|
+
* - Line chart by week
|
|
64
|
+
* - Expected: steady growth in cached calls over the dataset
|
|
65
|
+
*
|
|
66
|
+
* REAL-WORLD ANALOGUE: Prompt caching avoids re-processing long
|
|
67
|
+
* system prompts on every call, dramatically reducing cost and latency.
|
|
68
|
+
*
|
|
69
|
+
* ---------------------------------------------------------------
|
|
70
|
+
* 2. MODEL MIGRATION WAVE (TIMED RELEASE — event)
|
|
71
|
+
* ---------------------------------------------------------------
|
|
72
|
+
*
|
|
73
|
+
* PATTERN: At day 60, new model "opus-4-7" releases. After day 60,
|
|
74
|
+
* 35% of api calls from Build/Enterprise users switch model to
|
|
75
|
+
* "opus-4-7". These calls use 1.5x tokens (smarter model, longer
|
|
76
|
+
* responses). "opus-4-7" is part of the DECLARED model enum
|
|
77
|
+
* (schema-first: hooks only write declared values); the hook rewrites
|
|
78
|
+
* engine-sampled opus-4-7 back to the pre-release mix, so the model
|
|
79
|
+
* exists ONLY after day 60 and ONLY on Build/Enterprise api calls —
|
|
80
|
+
* exact purity, asserted in the story.
|
|
81
|
+
*
|
|
82
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
83
|
+
*
|
|
84
|
+
* Report 1: Model Distribution Over Time
|
|
85
|
+
* - Report type: Insights
|
|
86
|
+
* - Event: "api call"
|
|
87
|
+
* - Measure: Total
|
|
88
|
+
* - Breakdown: "model"
|
|
89
|
+
* - Line chart by week
|
|
90
|
+
* - Expected: opus-4-7 appears at day 60, ramps to ~35% of paid calls
|
|
91
|
+
*
|
|
92
|
+
* Report 2: Tokens Per Model
|
|
93
|
+
* - Report type: Insights
|
|
94
|
+
* - Event: "api call"
|
|
95
|
+
* - Measure: Average of "tokens_used"
|
|
96
|
+
* - Breakdown: "model"
|
|
97
|
+
* - Expected: opus-4-7 ~ 1.5x tokens vs other models. Clean on
|
|
98
|
+
* non-agentic/non-batch users — Hooks 3 and 7 multiply tokens_used
|
|
99
|
+
* on their cohorts and blur the comparison if left in.
|
|
100
|
+
*
|
|
101
|
+
* REAL-WORLD ANALOGUE: New flagship model launches cause migration
|
|
102
|
+
* waves among power users who want improved capabilities.
|
|
103
|
+
*
|
|
104
|
+
* ---------------------------------------------------------------
|
|
105
|
+
* 3. AGENTIC LOOP POWER USERS (everything)
|
|
106
|
+
* ---------------------------------------------------------------
|
|
107
|
+
*
|
|
108
|
+
* PATTERN: Users with 3+ "tool use call" AND 3+ api-call events with
|
|
109
|
+
* multi_turn=true — counted on the post-churn stream (Hooks 4 and 8 run
|
|
110
|
+
* first, so the cohort is exactly rebuildable from the output) — get 8x
|
|
111
|
+
* tokens_used on api calls plus 2 extra cloned api-call events per
|
|
112
|
+
* existing (3x volume). Clones carry fresh insert_ids (Mixpanel dedupes
|
|
113
|
+
* on $insert_id — spread-cloning the template id would silently drop
|
|
114
|
+
* every clone at import), unique offset timestamps, and multi_turn=true,
|
|
115
|
+
* which pushes the cohort's multi-turn share to (0.25n + 2n)/3n ≈ 75%
|
|
116
|
+
* vs ~25% baseline — a verifier-visible signature of the 3x volume.
|
|
117
|
+
* COMPOUNDS with Hook 7 (deliberate): agentic ∩ batch users get
|
|
118
|
+
* 8x × 2x = 16x tokens_used — agentic batch workloads are the
|
|
119
|
+
* platform's whales. No flag — discover via cohort builder.
|
|
120
|
+
*
|
|
121
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
122
|
+
*
|
|
123
|
+
* Report 1: Tokens per User — Agentic Cohort
|
|
124
|
+
* - Report type: Insights (with cohort)
|
|
125
|
+
* - Cohort A: users with >= 3 "tool use call" AND >= 3 api-call with multi_turn=true
|
|
126
|
+
* - Cohort B: rest
|
|
127
|
+
* - Event: "api call"
|
|
128
|
+
* - Measure: Average of "tokens_used"
|
|
129
|
+
* - Expected: A ~ 8x B (16x for the batch overlap — exclude users
|
|
130
|
+
* with a "batch job submitted" from both cohorts for the clean 8x)
|
|
131
|
+
*
|
|
132
|
+
* REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
|
|
133
|
+
* tokens via extended tool-use loops.
|
|
134
|
+
*
|
|
135
|
+
* ---------------------------------------------------------------
|
|
136
|
+
* 4. RATE LIMIT CHURN (everything)
|
|
137
|
+
* ---------------------------------------------------------------
|
|
138
|
+
*
|
|
139
|
+
* PATTERN: Users with >= 2 "rate limit error" events in first 7 days
|
|
140
|
+
* churn at 60%: a churned user's ENTIRE post-week-1 stream is dropped
|
|
141
|
+
* (retention cliff), the surviving 40% keep everything. Per-user, not
|
|
142
|
+
* per-event thinning — thinning is unverifiable on a burst-selected
|
|
143
|
+
* cohort (any pre/post ratio inherits the selection week's decay; any
|
|
144
|
+
* cross-user ratio inherits activity selection; measured RoR landed at
|
|
145
|
+
* 0.17 vs the 0.4 knob even stratified on week-1 activity). The cliff
|
|
146
|
+
* gives a selection-free proportion instead: share of flagged users
|
|
147
|
+
* with ZERO post-week-1 events ≈ 0.60 (minus a tiny natural-quiet
|
|
148
|
+
* baseline, which the story cancels by differencing against the
|
|
149
|
+
* unflagged share). Classification basis is pre-week-1 and survives
|
|
150
|
+
* every drop in the file, so the cohort is exactly rebuildable from
|
|
151
|
+
* output. No flag — discover via cohort.
|
|
152
|
+
*
|
|
153
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
154
|
+
*
|
|
155
|
+
* Report 1: Retention by Early Rate-Limit Cohort
|
|
156
|
+
* - Report type: Retention
|
|
157
|
+
* - Cohort A: users with >= 2 "rate limit error" in first 7 days
|
|
158
|
+
* - Cohort B: rest
|
|
159
|
+
* - Expected: cohort A's retention collapses to ~40% of cohort B's
|
|
160
|
+
* from week 2 onward — a hard cliff, not a gradual decay. ~60% of
|
|
161
|
+
* cohort A never appears again after their first week.
|
|
162
|
+
*
|
|
163
|
+
* REAL-WORLD ANALOGUE: Developers who get rate-limited early often
|
|
164
|
+
* switch to a competitor — and when they go, they go completely.
|
|
165
|
+
*
|
|
166
|
+
* ---------------------------------------------------------------
|
|
167
|
+
* 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
|
|
168
|
+
* ---------------------------------------------------------------
|
|
169
|
+
*
|
|
170
|
+
* PATTERN: Free users have context_window=200000, Build=1000000,
|
|
171
|
+
* Enterprise=2000000. Enterprise users send 4x larger input_tokens.
|
|
172
|
+
* Context window and input tokens are scaled by tier.
|
|
173
|
+
*
|
|
174
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
175
|
+
*
|
|
176
|
+
* Report 1: Input Tokens by Tier
|
|
177
|
+
* - Report type: Insights
|
|
178
|
+
* - Event: "api call"
|
|
179
|
+
* - Measure: Average of "input_tokens"
|
|
180
|
+
* - Breakdown: "api_tier" (superProp)
|
|
181
|
+
* - Expected: Enterprise ~ 4x Free (Enterprise ~ 8K, Free ~ 2K)
|
|
182
|
+
*
|
|
183
|
+
* Report 2: Context Window by Tier
|
|
184
|
+
* - Report type: Insights
|
|
185
|
+
* - Event: "api call"
|
|
186
|
+
* - Measure: Average of "context_window"
|
|
187
|
+
* - Breakdown: "api_tier"
|
|
188
|
+
* - Expected: Free=200K, Build=1M, Enterprise=2M
|
|
189
|
+
*
|
|
190
|
+
* REAL-WORLD ANALOGUE: Enterprise customers pay for larger context
|
|
191
|
+
* windows and use them for long-document analysis and code review.
|
|
192
|
+
*
|
|
193
|
+
* ---------------------------------------------------------------
|
|
194
|
+
* 6. OUTAGE DAY (TIME-BASED — event)
|
|
195
|
+
* ---------------------------------------------------------------
|
|
196
|
+
*
|
|
197
|
+
* PATTERN: Days 40-41, is_error is set to true on 40% of api call
|
|
198
|
+
* events, error_type is set to a service error, latency_ms is tripled.
|
|
199
|
+
* Baseline api-call error rate is 0% BY SCHEMA (is_error declares
|
|
200
|
+
* [false]; "rate limit error" is a separate event) — the outage is the
|
|
201
|
+
* only source of api-call errors, so the window is exact: ~40% error
|
|
202
|
+
* share inside days 40-41, exactly zero outside.
|
|
203
|
+
*
|
|
204
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
205
|
+
*
|
|
206
|
+
* Report 1: Error Rate Over Time
|
|
207
|
+
* - Report type: Insights
|
|
208
|
+
* - Event: "api call"
|
|
209
|
+
* - Measure: Total
|
|
210
|
+
* - Filter: is_error = true
|
|
211
|
+
* - Line chart by day
|
|
212
|
+
* - Expected: two-day spike at days 40-41 (~40% of api calls),
|
|
213
|
+
* flat zero everywhere else
|
|
214
|
+
*
|
|
215
|
+
* Report 2: Error Types During Outage
|
|
216
|
+
* - Report type: Insights
|
|
217
|
+
* - Event: "api call"
|
|
218
|
+
* - Filter: is_error = true
|
|
219
|
+
* - Breakdown: "error_type"
|
|
220
|
+
* - Date range: days 40-41
|
|
221
|
+
* - Expected: service_overloaded and internal_server_error dominate
|
|
222
|
+
*
|
|
223
|
+
* REAL-WORLD ANALOGUE: API platforms experience periodic outages
|
|
224
|
+
* that spike error rates across all customers.
|
|
225
|
+
*
|
|
226
|
+
* ---------------------------------------------------------------
|
|
227
|
+
* 7. BATCH API DISCOUNT (everything)
|
|
228
|
+
* ---------------------------------------------------------------
|
|
229
|
+
*
|
|
230
|
+
* PATTERN: Users with any "batch job submitted" event (on the
|
|
231
|
+
* post-churn stream — Hooks 4 and 8 run first) get 50% lower
|
|
232
|
+
* cost_per_token on api calls + 2x tokens_used. Mutates raw props.
|
|
233
|
+
* cost_per_token is touched by NO other hook — clean 0.5x.
|
|
234
|
+
* tokens_used COMPOUNDS with Hook 3 (deliberate): agentic ∩ batch
|
|
235
|
+
* users get 2x × 8x = 16x — verified as its own cohort cell.
|
|
236
|
+
* No flag — discover via cohort builder.
|
|
237
|
+
*
|
|
238
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
239
|
+
*
|
|
240
|
+
* Report 1: Cost per Token by Batch Cohort
|
|
241
|
+
* - Report type: Insights (with cohort)
|
|
242
|
+
* - Cohort A: users with >= 1 "batch job submitted"
|
|
243
|
+
* - Cohort B: rest
|
|
244
|
+
* - Event: "api call"
|
|
245
|
+
* - Measure: Average of "cost_per_token"
|
|
246
|
+
* - Expected: A ~ 0.5x B
|
|
247
|
+
*
|
|
248
|
+
* REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
|
|
249
|
+
*
|
|
250
|
+
* ---------------------------------------------------------------
|
|
251
|
+
* 8. EVAL-DRIVEN RETENTION (everything)
|
|
252
|
+
* ---------------------------------------------------------------
|
|
253
|
+
*
|
|
254
|
+
* PATTERN: Users with any "eval job" in first 7 days keep all events.
|
|
255
|
+
* Non-eval users lose 75% of post-day-30 events (25% keep-rate). The
|
|
256
|
+
* classification basis is pre-week-1 and survives every drop in the
|
|
257
|
+
* file, so the cohort is exactly rebuildable from output. No flag —
|
|
258
|
+
* discover via retention cohort.
|
|
259
|
+
*
|
|
260
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
261
|
+
*
|
|
262
|
+
* Report 1: Retention by Early Eval Cohort
|
|
263
|
+
* - Report type: Retention
|
|
264
|
+
* - Cohort A: users with >= 1 "eval job" in first 7 days
|
|
265
|
+
* - Cohort B: rest
|
|
266
|
+
* - Expected: cohort B's post-day-30 event volume (relative to its
|
|
267
|
+
* own first-30-day volume) runs ~0.25x cohort A's — a hard drop in
|
|
268
|
+
* B's retention curve after day 30. The engineered constant is the
|
|
269
|
+
* 0.25 volume ratio-of-ratios, not a specific D30 percentage.
|
|
270
|
+
*
|
|
271
|
+
* REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
|
|
272
|
+
*
|
|
273
|
+
* ---------------------------------------------------------------
|
|
274
|
+
* 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
275
|
+
* ---------------------------------------------------------------
|
|
276
|
+
*
|
|
277
|
+
* PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
|
|
278
|
+
* 2x faster than baseline (factor 0.5 on inter-event gaps); Free users
|
|
279
|
+
* 2x slower (factor 2.0). Mutates funnel event timestamps.
|
|
280
|
+
*
|
|
281
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
282
|
+
*
|
|
283
|
+
* Report 1: API to Eval — Median Time-to-Convert by Tier
|
|
284
|
+
* - Report type: Funnels
|
|
285
|
+
* - Steps: "api call" -> "tool use call" -> "eval job"
|
|
286
|
+
* - Measure: Median time to convert
|
|
287
|
+
* - Breakdown: "api_tier"
|
|
288
|
+
* - Expected: Enterprise < Build < Free, Enterprise well under
|
|
289
|
+
* Build's median, Free well over. The measured ratios sit between
|
|
290
|
+
* the pure factors (0.5x / 2.0x) and 1: greedy first-match funnel
|
|
291
|
+
* evaluation (Mixpanel's and the emulator's) pairs organic events
|
|
292
|
+
* into instances, diluting toward 1 — and dilution is asymmetric
|
|
293
|
+
* (stretched Free gaps intercept more organic events than
|
|
294
|
+
* compressed Enterprise gaps). The factors are 0.5/2.0 precisely
|
|
295
|
+
* so the report-visible signal survives that dilution.
|
|
296
|
+
*
|
|
297
|
+
* NOTE (funnel-post measurement): visible via funnel median TTC
|
|
298
|
+
* (Mixpanel report or emulateBreakdown timeToConvert). Cross-event
|
|
299
|
+
* MIN→MIN SQL on raw events does NOT show this — funnel-post adjusts
|
|
300
|
+
* gaps within funnel instances, not across the user's full history.
|
|
301
|
+
*
|
|
302
|
+
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
|
|
303
|
+
* who execute end-to-end pipelines faster.
|
|
304
|
+
*
|
|
305
|
+
* ---------------------------------------------------------------
|
|
306
|
+
* 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
|
|
307
|
+
* ---------------------------------------------------------------
|
|
308
|
+
*
|
|
309
|
+
* PATTERN: Count "docs searched" events strictly between the EARLIEST
|
|
310
|
+
* organization-created and the EARLIEST billing-payment (by time, not
|
|
311
|
+
* array order). Sweet 1-2 → amount_usd × 1.35 on ALL billing payments.
|
|
312
|
+
* Over 3+ → amount_usd × 0.75 on ALL billing payments. Zero docs →
|
|
313
|
+
* untouched baseline. Only born-in-dataset users have an organization
|
|
314
|
+
* created event AND a billing payment (~4.6% of users), so thresholds
|
|
315
|
+
* are calibrated to the real docs_ct distribution in that segment
|
|
316
|
+
* (measured 0/1/2/3/4/5+ ≈ 43/19/14/12/10/2 per-cent — the old 5+
|
|
317
|
+
* "over" bin held ~7 users at full fidelity, a dead branch). Both
|
|
318
|
+
* effects are amount mutations, deliberately: amount_usd draws iid
|
|
319
|
+
* from the declared distribution regardless of user activity, so
|
|
320
|
+
* median ratios recover the knobs selection-free — count effects on
|
|
321
|
+
* ~100-user cohorts drown in activity-selection bias (a placebo test
|
|
322
|
+
* on the untouched 3+ cohort read 1.22-1.30 under the best count
|
|
323
|
+
* normalizer we found). No flag.
|
|
324
|
+
*
|
|
325
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
326
|
+
*
|
|
327
|
+
* Report 1: Median Billing Amount by Docs-Searched Bucket
|
|
328
|
+
* - Report type: Insights (with cohort)
|
|
329
|
+
* - Cohort A: users with 1-2 "docs searched" between org creation and
|
|
330
|
+
* first billing; Cohort B: users with 0; Cohort C: users with 3+
|
|
331
|
+
* - Event: "billing payment"
|
|
332
|
+
* - Measure: Median of "amount_usd"
|
|
333
|
+
* - Expected: A ~ 1.35x B; C ~ 0.75x B
|
|
334
|
+
*
|
|
335
|
+
* REAL-WORLD ANALOGUE: A little docs reading lifts willingness to pay;
|
|
336
|
+
* doc obsession signals a team stuck on integration that downgrades.
|
|
337
|
+
*
|
|
338
|
+
* ===============================================================
|
|
339
|
+
* EXPECTED METRICS SUMMARY
|
|
340
|
+
* ===============================================================
|
|
341
|
+
*
|
|
342
|
+
* Hook | Metric | Expected | Measured (full)
|
|
343
|
+
* ----------------------------|-------------------------------------------|---------------|----------------
|
|
344
|
+
* H1 Prompt Caching | avg cost_usd cached/uncached | 0.3x | 0.3005
|
|
345
|
+
* H1 Prompt Caching | share of users with any cached call | ~25% | 0.2297
|
|
346
|
+
* H2 Model Migration | opus-4-7 pre-day-60 / Free / non-api-call | 0 (exact) | 0 of 35975
|
|
347
|
+
* H2 Model Migration | opus-4-7 share, paid calls post-day-60 | ~35% | 0.3484
|
|
348
|
+
* H2 Model Migration | tokens opus/other (non-agentic/non-batch) | 1.5x | 1.461
|
|
349
|
+
* H3 Agentic Power Users | avg tokens agentic-only / neither | 8x | 7.875
|
|
350
|
+
* H3 Agentic Power Users | avg tokens agentic∩batch / neither | 16x (with H7) | 15.97
|
|
351
|
+
* H3 Agentic Power Users | multi-turn share of agentic api calls | ~75% (3x vol) | 0.7644
|
|
352
|
+
* H4 Rate Limit Churn | zero-post-week-1 share, flagged − rest | ~+0.60 diff | +0.6072 (flagged 0.6079)
|
|
353
|
+
* H5 Tier Context Window | avg input_tokens Enterprise / Free | 4x | 3.984
|
|
354
|
+
* H5 Tier Context Window | context_window per tier | 200K/1M/2M | exact (min=max per tier)
|
|
355
|
+
* H6 Outage Day | api-call error share, days 40-41 | ~40% (0% out) | 0.3996 (0 out-of-window)
|
|
356
|
+
* H7 Batch Discount | avg cost_per_token batch / rest | 0.5x | 0.4968
|
|
357
|
+
* H7 Batch Discount | avg tokens batch-only / neither | 2x | 2.159
|
|
358
|
+
* H8 Eval Retention | post/pre day-30 volume, noneval vs eval | 0.25x RoR | 0.2552
|
|
359
|
+
* H9 API-to-Eval TTC | funnel median TTC: Ent/Build, Free/Build | <1 / >1 (0.5, 2.0 pure; diluted) | 0.6606 / 1.263 (emulator, 336h window)
|
|
360
|
+
* H10 Docs Magic Number | median amount_usd sweet(1-2) / zero | 1.35x | 1.456
|
|
361
|
+
* H10 Docs Magic Number | median amount_usd over(3+) / zero | 0.75x | 0.8371
|
|
362
|
+
*/
|
|
363
|
+
|
|
364
|
+
// ── SCALE ──
|
|
365
|
+
const SEED = "promptforge";
|
|
366
|
+
const NUM_USERS = 10_000;
|
|
367
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
368
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
369
|
+
const EVENTS_PER_DAY = 0.83;
|
|
370
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
371
|
+
|
|
372
|
+
const chance = u.initChance(SEED);
|
|
373
|
+
|
|
374
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
375
|
+
const OUTAGE_START_DAY = 40;
|
|
376
|
+
const OUTAGE_END_DAY = 42;
|
|
377
|
+
const OUTAGE_ERROR_LIKELIHOOD = 40;
|
|
378
|
+
const OUTAGE_LATENCY_MULT = 3;
|
|
379
|
+
|
|
380
|
+
const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
|
|
381
|
+
const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
|
|
382
|
+
|
|
383
|
+
const CACHE_USER_HASH_MOD = 4;
|
|
384
|
+
const CACHE_COST_FACTOR = 0.3;
|
|
385
|
+
const CACHE_ACTIVATION_PCT = 0.2;
|
|
386
|
+
|
|
387
|
+
const MODEL_MIGRATION_DAY = 60;
|
|
388
|
+
const MODEL_MIGRATION_LIKELIHOOD = 35;
|
|
389
|
+
const MODEL_MIGRATION_TOKEN_MULT = 1.5;
|
|
390
|
+
// pre-release model mix — the declared api-call enum minus opus-4-7. H2
|
|
391
|
+
// rewrites engine-sampled opus-4-7 back to this mix so the model exists only
|
|
392
|
+
// after release day (schema-first requires opus-4-7 in the declared enum).
|
|
393
|
+
const PRE_RELEASE_MODELS = ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"];
|
|
394
|
+
|
|
395
|
+
const AGENTIC_TOOL_THRESHOLD = 3;
|
|
396
|
+
const AGENTIC_MULTITURN_THRESHOLD = 3;
|
|
397
|
+
const AGENTIC_TOKEN_MULT = 8;
|
|
398
|
+
const AGENTIC_CLONE_MULT = 2;
|
|
399
|
+
|
|
400
|
+
const RATE_LIMIT_THRESHOLD = 2;
|
|
401
|
+
// per-USER churn probability: a churned user loses ALL post-week-1 events
|
|
402
|
+
const RATE_LIMIT_CHURN_LIKELIHOOD = 60;
|
|
403
|
+
|
|
404
|
+
const BATCH_COST_FACTOR = 0.5;
|
|
405
|
+
const BATCH_TOKEN_MULT = 2;
|
|
406
|
+
|
|
407
|
+
const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
|
|
408
|
+
const EVAL_CUTOFF_DAYS = 30;
|
|
409
|
+
|
|
410
|
+
// bins calibrated to the measured docs_ct distribution among org∩billing
|
|
411
|
+
// users (~4.6% of users): 0 ≈ 43%, 1-2 ≈ 33%, 3+ ≈ 24% — every cohort
|
|
412
|
+
// clears ~100 users at full fidelity (the old 5+ bin held ~7: dead branch)
|
|
413
|
+
const DOCS_SWEET_MIN = 1;
|
|
414
|
+
const DOCS_SWEET_MAX = 2;
|
|
415
|
+
const DOCS_OVER_THRESHOLD = 3;
|
|
416
|
+
const DOCS_BILLING_BOOST = 1.35;
|
|
417
|
+
const DOCS_OVER_PENALTY = 0.75;
|
|
418
|
+
|
|
419
|
+
// 0.5/2.0 (not the 1.5 file's 0.67/1.4): greedy first-match funnel pairing
|
|
420
|
+
// dilutes measured TTC ratios toward 1 — these factors keep the report-
|
|
421
|
+
// visible signal clear of noise after dilution
|
|
422
|
+
const FUNNEL_TTC_ENTERPRISE = 0.5;
|
|
423
|
+
const FUNNEL_TTC_FREE = 2.0;
|
|
424
|
+
|
|
425
|
+
// ── HELPER FUNCTIONS ──
|
|
426
|
+
function handleFunnelPostHooks(record, meta) {
|
|
427
|
+
// H9: API-to-Eval TTC scaled by tier
|
|
428
|
+
const segment = meta?.profile?.api_tier;
|
|
429
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
430
|
+
const factor = (
|
|
431
|
+
segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
|
|
432
|
+
segment === "Free" ? FUNNEL_TTC_FREE :
|
|
433
|
+
1.0
|
|
434
|
+
);
|
|
435
|
+
if (factor !== 1.0) {
|
|
436
|
+
for (let i = 1; i < record.length; i++) {
|
|
437
|
+
const prev = dayjs(record[i - 1].time);
|
|
438
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
439
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
return record;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function handleEverythingHooks(record, meta) {
|
|
447
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
448
|
+
let events = record;
|
|
449
|
+
if (!events.length) return record;
|
|
450
|
+
const profile = meta && meta.profile ? meta.profile : {};
|
|
451
|
+
|
|
452
|
+
// ── ORDERING ──
|
|
453
|
+
// types.d.ts (HookMetaEverything) recommends stamp → mutate/clone → filter →
|
|
454
|
+
// temporal. This file deliberately diverges in ONE spot, documented here: the
|
|
455
|
+
// cohort-classified mutators (H3 agentic, H7 batch, H10 docs) run AFTER the
|
|
456
|
+
// filters (H4, H8) so each cohort's classification basis is exactly the
|
|
457
|
+
// surviving event stream. Classifying pre-drop makes the cohort unrecoverable
|
|
458
|
+
// from the output (the verifier cannot see dropped events) — the 1.5 file
|
|
459
|
+
// classified H3 before H4/H8's drops and its verifier "confirmed" 8x against
|
|
460
|
+
// a leaky cohort at 1.5x. None of H3/H7/H10 anchors on a dataset-day window,
|
|
461
|
+
// so post-filter classification costs nothing temporally. H6 (outage window)
|
|
462
|
+
// runs LAST per the types.d.ts rule so H3's clones landing inside days 40-41
|
|
463
|
+
// get error-stamped like every other call.
|
|
464
|
+
|
|
465
|
+
// ── PHASE 1: stamps ──
|
|
466
|
+
events.forEach(e => {
|
|
467
|
+
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
468
|
+
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
469
|
+
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
// H5: Tier-based context window & input tokens
|
|
473
|
+
const tier = profile.api_tier || "Free";
|
|
474
|
+
const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
|
|
475
|
+
const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
|
|
476
|
+
events.forEach(e => {
|
|
477
|
+
if (e.event === "api call") {
|
|
478
|
+
e.context_window = contextWindow;
|
|
479
|
+
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
// H1: Prompt caching adoption — ~25% of users; activates ~20% into stream.
|
|
484
|
+
// Hash the PROFILE distinct_id, not events[0].user_id: the 1.5 file hashed
|
|
485
|
+
// events[0].user_id, which is undefined on device-only records —
|
|
486
|
+
// String(undefined || "") reduces to 0 and 0 % 4 === 0, silently classifying
|
|
487
|
+
// every such user as a cache user.
|
|
488
|
+
const hashBasis = String(profile.distinct_id || (events.find(e => e.user_id) || {}).user_id || "");
|
|
489
|
+
const idHash = hashBasis.split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
490
|
+
const isCacheUser = hashBasis.length > 0 && (idHash % CACHE_USER_HASH_MOD) === 0;
|
|
491
|
+
if (isCacheUser) {
|
|
492
|
+
let cacheActivated = false;
|
|
493
|
+
const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
|
|
494
|
+
events.forEach((e, idx) => {
|
|
495
|
+
if (e.event === "api call") {
|
|
496
|
+
if (idx >= activationPoint) cacheActivated = true;
|
|
497
|
+
if (cacheActivated) {
|
|
498
|
+
e.cache_enabled = true;
|
|
499
|
+
e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// H2: Model migration wave.
|
|
506
|
+
// "opus-4-7" is in the DECLARED model enum (schema-first: hooks only write
|
|
507
|
+
// declared values), which means the engine samples it uniformly across the
|
|
508
|
+
// whole window — but the story needs zero opus-4-7 before release day.
|
|
509
|
+
// Rewrite every engine-sampled opus-4-7 back to the pre-release mix first;
|
|
510
|
+
// opus-4-7 in the output therefore comes from the migration stamp alone
|
|
511
|
+
// (post-day-60, Build/Enterprise api calls only — exact purity, asserted).
|
|
512
|
+
events.forEach(e => {
|
|
513
|
+
if (e.event === "api call" && e.model === "opus-4-7") {
|
|
514
|
+
e.model = chance.pickone(PRE_RELEASE_MODELS);
|
|
515
|
+
}
|
|
516
|
+
});
|
|
517
|
+
const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
|
|
518
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
519
|
+
events.forEach(e => {
|
|
520
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
|
|
521
|
+
if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
|
|
522
|
+
e.model = "opus-4-7";
|
|
523
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// First-event anchor for both churn filters. Computed pre-filter, but always
|
|
530
|
+
// verifier-recoverable: the filters only drop events STRICTLY after
|
|
531
|
+
// t0 + 7d / t0 + 30d, so min(time) in the output still equals t0.
|
|
532
|
+
const firstEventTime = events.reduce((min, e) => {
|
|
533
|
+
const t = dayjs(e.time);
|
|
534
|
+
return t.isBefore(min) ? t : min;
|
|
535
|
+
}, dayjs(events[0].time));
|
|
536
|
+
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
537
|
+
|
|
538
|
+
// ── PHASE 2: filters ──
|
|
539
|
+
// H4: Rate-limit churn — 2+ early rate-limit errors → 60% of flagged users
|
|
540
|
+
// lose their ENTIRE post-week-1 stream. Per-USER cliff, not per-event
|
|
541
|
+
// thinning: thinning is unverifiable on a burst-selected cohort (pre/post
|
|
542
|
+
// ratios inherit the selection week's decay; cross-user ratios inherit
|
|
543
|
+
// activity selection), while the cliff yields a selection-free proportion —
|
|
544
|
+
// share of flagged users with zero post-week-1 events ≈ 0.6. Classification
|
|
545
|
+
// basis is pre-week-1 and survives every drop.
|
|
546
|
+
const earlyRateLimits = events.filter(e =>
|
|
547
|
+
e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
548
|
+
).length;
|
|
549
|
+
if (earlyRateLimits >= RATE_LIMIT_THRESHOLD && chance.bool({ likelihood: RATE_LIMIT_CHURN_LIKELIHOOD })) {
|
|
550
|
+
events = events.filter(e => !dayjs(e.time).isAfter(firstWeekEnd));
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// H8: Eval-driven retention — non-eval users lose 75% of post-day-30 events
|
|
554
|
+
// (classification basis is pre-week-1 and survives every drop)
|
|
555
|
+
const hasEarlyEval = events.some(e =>
|
|
556
|
+
e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
557
|
+
);
|
|
558
|
+
if (!hasEarlyEval) {
|
|
559
|
+
const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
|
|
560
|
+
events = events.filter(e => {
|
|
561
|
+
if (dayjs(e.time).isAfter(cutoff)) {
|
|
562
|
+
return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
|
|
563
|
+
}
|
|
564
|
+
return true;
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// ── PHASE 3: cohort mutators + clones (classified on the SURVIVING stream) ──
|
|
569
|
+
// H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens,
|
|
570
|
+
// 2 clones per surviving api call (3x volume). Clones need FRESH insert_ids:
|
|
571
|
+
// the engine stamps insert_id at generation (lib/generators/events.js), so a
|
|
572
|
+
// bare spread copies the template's id and Mixpanel's $insert_id dedupe
|
|
573
|
+
// silently drops every clone at import — the 1.5 file shipped that bug.
|
|
574
|
+
// Clones stamp multi_turn: true, pushing the cohort's multi-turn share to
|
|
575
|
+
// (0.25n + 2n)/3n ≈ 75% — the verifier-visible signature of the 3x volume.
|
|
576
|
+
// Classification stays exactly recoverable from output: clones only ADD
|
|
577
|
+
// multi-turn api calls to users already at/above both thresholds.
|
|
578
|
+
// COMPOUND (deliberate): H7 below also multiplies tokens_used on these same
|
|
579
|
+
// events, clones included — agentic ∩ batch users land at 8 × 2 = 16x. The
|
|
580
|
+
// story verifies all four cells (neither/agentic/batch/both = 1x/8x/2x/16x).
|
|
581
|
+
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
582
|
+
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
583
|
+
const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
|
|
584
|
+
if (isAgenticUser) {
|
|
585
|
+
events.forEach(e => {
|
|
586
|
+
if (e.event === "api call") {
|
|
587
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
|
|
588
|
+
}
|
|
589
|
+
});
|
|
590
|
+
const apiCalls = events.filter(e => e.event === "api call");
|
|
591
|
+
const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
|
|
592
|
+
for (let i = 0; i < extraCount; i++) {
|
|
593
|
+
const template = apiCalls[i % apiCalls.length];
|
|
594
|
+
if (template) {
|
|
595
|
+
events.push({
|
|
596
|
+
...template,
|
|
597
|
+
insert_id: chance.guid(),
|
|
598
|
+
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
599
|
+
user_id: template.user_id,
|
|
600
|
+
multi_turn: true,
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// H7: Batch API discount — any surviving batch job submitted → 0.5x
|
|
607
|
+
// cost_per_token (touched by NO other hook — clean), 2x tokens_used
|
|
608
|
+
// (COMPOUNDS with H3, see above). Runs after H3 so the clones get the
|
|
609
|
+
// discount too — a batch user's api calls are uniformly discounted.
|
|
610
|
+
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
611
|
+
if (isBatchUser) {
|
|
612
|
+
events.forEach(e => {
|
|
613
|
+
if (e.event === "api call") {
|
|
614
|
+
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
|
|
615
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
// H10: Docs-searched magic number — docs strictly between the EARLIEST
|
|
621
|
+
// org-created and the EARLIEST billing payment (by time — the 1.5 file used
|
|
622
|
+
// Array.find, i.e. array order, on a not-yet-sorted stream). Both branches
|
|
623
|
+
// mutate amount_usd only (sweet ×1.35, over ×0.75): amounts draw iid from
|
|
624
|
+
// the declared distribution, so median ratios recover the knobs selection-
|
|
625
|
+
// free, and nothing is dropped — the classification window is always fully
|
|
626
|
+
// reconstructable from output.
|
|
627
|
+
const orgEvent = events.reduce((min, e) =>
|
|
628
|
+
e.event === "organization created" && (!min || dayjs(e.time).isBefore(dayjs(min.time))) ? e : min, null);
|
|
629
|
+
const firstBilling = events.reduce((min, e) =>
|
|
630
|
+
e.event === "billing payment" && (!min || dayjs(e.time).isBefore(dayjs(min.time))) ? e : min, null);
|
|
631
|
+
if (orgEvent && firstBilling) {
|
|
632
|
+
const aTime = dayjs(orgEvent.time);
|
|
633
|
+
const bTime = dayjs(firstBilling.time);
|
|
634
|
+
const docsBetween = events.filter(e =>
|
|
635
|
+
e.event === "docs searched" &&
|
|
636
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
637
|
+
dayjs(e.time).isBefore(bTime)
|
|
638
|
+
).length;
|
|
639
|
+
if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
|
|
640
|
+
events.forEach(e => {
|
|
641
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
642
|
+
e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
|
|
643
|
+
}
|
|
644
|
+
});
|
|
645
|
+
} else if (docsBetween >= DOCS_OVER_THRESHOLD) {
|
|
646
|
+
events.forEach(e => {
|
|
647
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
648
|
+
e.amount_usd = Math.round(e.amount_usd * DOCS_OVER_PENALTY);
|
|
649
|
+
}
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// ── PHASE 4: temporal mutation LAST (clones in the window get stamped too) ──
|
|
655
|
+
// H6: Outage window [day 40, day 42) — 40% of api calls flagged is_error
|
|
656
|
+
// with a service error_type and 3x latency. Baseline is_error is 0% by
|
|
657
|
+
// schema (declared [false]), so the outage is the ONLY source of api-call
|
|
658
|
+
// errors: ~40% share inside the window, exactly zero outside.
|
|
659
|
+
events.forEach(e => {
|
|
660
|
+
if (e.event !== "api call") return;
|
|
661
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
662
|
+
if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
|
|
663
|
+
if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
|
|
664
|
+
e.is_error = true;
|
|
665
|
+
e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
|
|
666
|
+
e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
return events;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// ── CONFIG ──
|
|
675
|
+
/** @type {Config} */
|
|
676
|
+
const config = {
|
|
677
|
+
version: 2,
|
|
678
|
+
seed: SEED,
|
|
679
|
+
datasetStart: DATASET_START,
|
|
680
|
+
datasetEnd: DATASET_END,
|
|
681
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
682
|
+
numUsers: NUM_USERS,
|
|
683
|
+
format: "json",
|
|
684
|
+
gzip: true,
|
|
685
|
+
credentials: {
|
|
686
|
+
token,
|
|
687
|
+
},
|
|
688
|
+
switches: {
|
|
689
|
+
hasSessionIds: true,
|
|
690
|
+
alsoInferFunnels: false,
|
|
691
|
+
hasLocation: true,
|
|
692
|
+
hasAndroidDevices: false,
|
|
693
|
+
hasIOSDevices: false,
|
|
694
|
+
hasDesktopDevices: true,
|
|
695
|
+
hasBrowser: true,
|
|
696
|
+
hasCampaigns: false,
|
|
697
|
+
isAnonymous: false,
|
|
698
|
+
hasAdSpend: false,
|
|
699
|
+
hasAvatar: true,
|
|
700
|
+
},
|
|
701
|
+
identity: {
|
|
702
|
+
avgDevicePerUser: 2,
|
|
703
|
+
},
|
|
704
|
+
concurrency: 1,
|
|
705
|
+
writeToDisk: false,
|
|
706
|
+
soup: "growth",
|
|
707
|
+
scdProps: {
|
|
708
|
+
monthly_api_usage: {
|
|
709
|
+
values: u.weighNumRange(0, 1000000, 0.3, 50),
|
|
710
|
+
frequency: "week",
|
|
711
|
+
timing: "fuzzy",
|
|
712
|
+
max: 20,
|
|
713
|
+
},
|
|
714
|
+
api_tier_history: {
|
|
715
|
+
values: ["Free", "Build", "Enterprise"],
|
|
716
|
+
frequency: "month",
|
|
717
|
+
timing: "fixed",
|
|
718
|
+
max: 6,
|
|
719
|
+
},
|
|
720
|
+
},
|
|
721
|
+
events: [
|
|
722
|
+
{
|
|
723
|
+
event: "organization created",
|
|
724
|
+
weight: 1,
|
|
725
|
+
isFirstEvent: true,
|
|
726
|
+
isAuthEvent: true,
|
|
727
|
+
properties: {
|
|
728
|
+
org_size: ["solo", "startup", "growth", "enterprise"],
|
|
729
|
+
referral_source: ["docs", "blog", "github", "word_of_mouth", "search", "conference"],
|
|
730
|
+
},
|
|
731
|
+
},
|
|
732
|
+
{
|
|
733
|
+
event: "api key created",
|
|
734
|
+
weight: 2,
|
|
735
|
+
properties: {
|
|
736
|
+
key_type: ["development", "production", "staging"],
|
|
737
|
+
key_scope: ["full_access", "read_only", "completions_only"],
|
|
738
|
+
},
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
event: "api key rotated",
|
|
742
|
+
weight: 1,
|
|
743
|
+
properties: {
|
|
744
|
+
rotation_reason: ["scheduled", "compromised", "policy", "manual"],
|
|
745
|
+
},
|
|
746
|
+
},
|
|
747
|
+
{
|
|
748
|
+
event: "api call",
|
|
749
|
+
weight: 10,
|
|
750
|
+
isStrictEvent: false,
|
|
751
|
+
properties: {
|
|
752
|
+
// opus-4-7 is declared so the H2 hook only writes declared values
|
|
753
|
+
// (schema-first); the hook scrubs engine-sampled opus-4-7 back to
|
|
754
|
+
// the pre-release mix, so it appears ONLY via the day-60 migration
|
|
755
|
+
model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6", "opus-4-7"],
|
|
756
|
+
input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
|
|
757
|
+
output_tokens: u.weighNumRange(10, 4000, 0.4, 500),
|
|
758
|
+
tokens_used: u.weighNumRange(100, 12000, 0.4, 2500),
|
|
759
|
+
cost_usd: [0.001, 0.002, 0.003, 0.003, 0.005, 0.005, 0.005, 0.008, 0.008, 0.01, 0.01, 0.01, 0.01, 0.015, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
|
|
760
|
+
cost_per_token: [0.000002, 0.000003, 0.000005, 0.000005, 0.000008, 0.000008, 0.00001, 0.00001, 0.00001, 0.000012, 0.000015, 0.00002, 0.000025, 0.00003],
|
|
761
|
+
latency_ms: u.weighNumRange(100, 15000, 0.4, 1500),
|
|
762
|
+
cache_enabled: [false],
|
|
763
|
+
is_error: [false],
|
|
764
|
+
error_type: ["none"],
|
|
765
|
+
multi_turn: [false, false, false, true],
|
|
766
|
+
context_window: [200000],
|
|
767
|
+
stream: [true, true, true, false],
|
|
768
|
+
stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
|
|
769
|
+
},
|
|
770
|
+
},
|
|
771
|
+
{
|
|
772
|
+
event: "tool use call",
|
|
773
|
+
weight: 4,
|
|
774
|
+
isStrictEvent: false,
|
|
775
|
+
properties: {
|
|
776
|
+
tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
|
|
777
|
+
execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
|
|
778
|
+
success: [true, true, true, true, false],
|
|
779
|
+
tool_input_tokens: u.weighNumRange(50, 2000, 0.4, 300),
|
|
780
|
+
tool_output_tokens: u.weighNumRange(20, 5000, 0.4, 500),
|
|
781
|
+
},
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
event: "batch job submitted",
|
|
785
|
+
weight: 2,
|
|
786
|
+
properties: {
|
|
787
|
+
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
788
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
789
|
+
estimated_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
790
|
+
priority: ["standard", "standard", "standard", "express"],
|
|
791
|
+
},
|
|
792
|
+
},
|
|
793
|
+
{
|
|
794
|
+
event: "batch job completed",
|
|
795
|
+
weight: 2,
|
|
796
|
+
properties: {
|
|
797
|
+
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
798
|
+
processing_time_sec: u.weighNumRange(60, 7200, 0.4, 900),
|
|
799
|
+
total_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
800
|
+
success_rate: u.weighNumRange(90, 100, 0.8, 98),
|
|
801
|
+
},
|
|
802
|
+
},
|
|
803
|
+
{
|
|
804
|
+
event: "eval job",
|
|
805
|
+
weight: 3,
|
|
806
|
+
isStrictEvent: false,
|
|
807
|
+
properties: {
|
|
808
|
+
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
809
|
+
num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
|
|
810
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
811
|
+
dataset_name: ["prod_prompts", "safety_suite", "regression_set", "benchmark_v2", "custom_eval"],
|
|
812
|
+
},
|
|
813
|
+
},
|
|
814
|
+
{
|
|
815
|
+
event: "eval result",
|
|
816
|
+
weight: 3,
|
|
817
|
+
properties: {
|
|
818
|
+
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
819
|
+
score: u.weighNumRange(0, 100, 0.6, 75),
|
|
820
|
+
pass_rate: u.weighNumRange(50, 100, 0.7, 85),
|
|
821
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
822
|
+
regression_detected: [false, false, false, false, true],
|
|
823
|
+
},
|
|
824
|
+
},
|
|
825
|
+
{
|
|
826
|
+
event: "rate limit error",
|
|
827
|
+
weight: 3,
|
|
828
|
+
properties: {
|
|
829
|
+
error_code: [429],
|
|
830
|
+
retry_after_ms: u.weighNumRange(1000, 60000, 0.3, 5000),
|
|
831
|
+
requests_per_minute: u.weighNumRange(50, 2000, 0.4, 500),
|
|
832
|
+
tier_limit: ["Free", "Build", "Enterprise"],
|
|
833
|
+
},
|
|
834
|
+
},
|
|
835
|
+
{
|
|
836
|
+
event: "billing payment",
|
|
837
|
+
weight: 2,
|
|
838
|
+
isStrictEvent: false,
|
|
839
|
+
properties: {
|
|
840
|
+
amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
|
|
841
|
+
payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
|
|
842
|
+
billing_period: ["monthly", "monthly", "annual"],
|
|
843
|
+
tokens_consumed: u.weighNumRange(100000, 50000000, 0.3, 5000000),
|
|
844
|
+
},
|
|
845
|
+
},
|
|
846
|
+
{
|
|
847
|
+
event: "model selected",
|
|
848
|
+
weight: 3,
|
|
849
|
+
properties: {
|
|
850
|
+
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
851
|
+
is_default: [true, true, false],
|
|
852
|
+
selection_context: ["playground", "api_config", "eval_setup", "batch_config"],
|
|
853
|
+
},
|
|
854
|
+
},
|
|
855
|
+
{
|
|
856
|
+
event: "dashboard viewed",
|
|
857
|
+
weight: 5,
|
|
858
|
+
properties: {
|
|
859
|
+
dashboard_section: ["usage", "billing", "api_keys", "models", "evals", "logs"],
|
|
860
|
+
time_range: ["1h", "24h", "7d", "30d"],
|
|
861
|
+
},
|
|
862
|
+
},
|
|
863
|
+
{
|
|
864
|
+
event: "docs searched",
|
|
865
|
+
weight: 4,
|
|
866
|
+
properties: {
|
|
867
|
+
search_query_category: ["api_reference", "quickstart", "pricing", "models", "tool_use", "batch_api", "caching", "errors"],
|
|
868
|
+
results_found: u.weighNumRange(0, 50, 0.5, 8),
|
|
869
|
+
clicked_result: [true, true, true, false],
|
|
870
|
+
},
|
|
871
|
+
},
|
|
872
|
+
{
|
|
873
|
+
event: "member invited",
|
|
874
|
+
weight: 2,
|
|
875
|
+
properties: {
|
|
876
|
+
invite_role: ["admin", "developer", "developer", "billing", "viewer"],
|
|
877
|
+
invite_method: ["email", "email", "sso", "link"],
|
|
878
|
+
},
|
|
879
|
+
},
|
|
880
|
+
{
|
|
881
|
+
event: "webhook configured",
|
|
882
|
+
weight: 1,
|
|
883
|
+
properties: {
|
|
884
|
+
webhook_event: ["usage_alert", "rate_limit", "batch_complete", "eval_complete", "billing_threshold"],
|
|
885
|
+
delivery_method: ["https", "https", "slack", "email"],
|
|
886
|
+
},
|
|
887
|
+
},
|
|
888
|
+
{
|
|
889
|
+
event: "playground session",
|
|
890
|
+
weight: 4,
|
|
891
|
+
properties: {
|
|
892
|
+
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
893
|
+
turns: u.weighNumRange(1, 30, 0.4, 5),
|
|
894
|
+
shared: [false, false, false, true],
|
|
895
|
+
tokens_used: u.weighNumRange(100, 20000, 0.3, 3000),
|
|
896
|
+
},
|
|
897
|
+
},
|
|
898
|
+
{
|
|
899
|
+
event: "account deactivated",
|
|
900
|
+
weight: 1,
|
|
901
|
+
isChurnEvent: true,
|
|
902
|
+
returnLikelihood: 0.1,
|
|
903
|
+
isStrictEvent: true,
|
|
904
|
+
properties: {
|
|
905
|
+
reason: ["cost", "switched_provider", "project_ended", "rate_limits", "no_longer_needed", "performance"],
|
|
906
|
+
},
|
|
907
|
+
},
|
|
908
|
+
],
|
|
909
|
+
funnels: [
|
|
910
|
+
{
|
|
911
|
+
name: "Onboarding",
|
|
912
|
+
sequence: ["organization created", "api key created", "api call"],
|
|
913
|
+
conversionRate: 70,
|
|
914
|
+
order: "sequential",
|
|
915
|
+
isFirstFunnel: true,
|
|
916
|
+
timeToConvert: 48,
|
|
917
|
+
weight: 3,
|
|
918
|
+
},
|
|
919
|
+
{
|
|
920
|
+
name: "API to Eval Pipeline",
|
|
921
|
+
sequence: ["api call", "tool use call", "eval job"],
|
|
922
|
+
conversionRate: 45,
|
|
923
|
+
order: "sequential",
|
|
924
|
+
timeToConvert: 168,
|
|
925
|
+
weight: 5,
|
|
926
|
+
},
|
|
927
|
+
{
|
|
928
|
+
name: "Usage to Billing",
|
|
929
|
+
sequence: ["api call", "billing payment"],
|
|
930
|
+
conversionRate: 30,
|
|
931
|
+
order: "sequential",
|
|
932
|
+
timeToConvert: 336,
|
|
933
|
+
weight: 2,
|
|
934
|
+
},
|
|
935
|
+
],
|
|
936
|
+
superProps: {
|
|
937
|
+
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
938
|
+
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
939
|
+
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
940
|
+
},
|
|
941
|
+
userProps: {
|
|
942
|
+
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
943
|
+
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
944
|
+
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
945
|
+
monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
|
|
946
|
+
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
947
|
+
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
948
|
+
},
|
|
949
|
+
hook(record, type, meta) {
|
|
950
|
+
if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
|
|
951
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
952
|
+
return record;
|
|
953
|
+
},
|
|
954
|
+
};
|
|
955
|
+
|
|
956
|
+
// ── STORIES ──────────────────────────────────────────────────────────────
|
|
957
|
+
// Machine-checkable contract for the 10 numbered hooks. Evaluate with:
|
|
958
|
+
// node scripts/verify-stories.mjs dungeons/vertical/ai-platform/ai-platform.js --data-prefix verify-ai-platform
|
|
959
|
+
|
|
960
|
+
const EV = `read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true)`;
|
|
961
|
+
const US = `read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)`;
|
|
962
|
+
|
|
963
|
+
// Identity prelude. organization created is both isAuthEvent and isFirstEvent,
|
|
964
|
+
// so born users auth on their very first event and user_id should be present
|
|
965
|
+
// on every record; the prelude still resolves through the device pool
|
|
966
|
+
// (avgDevicePerUser: 2, "anonymousIds" is the legacy USERS-shard key) as
|
|
967
|
+
// belt-and-braces for any device-only edge.
|
|
968
|
+
const ID_CTE = `dmap AS (SELECT unnest("anonymousIds") AS device_id, distinct_id FROM ${US}),
|
|
969
|
+
ev AS (SELECT coalesce(m.distinct_id::VARCHAR, e.user_id::VARCHAR, e.device_id::VARCHAR) AS uid,
|
|
970
|
+
e.time::TIMESTAMP AS t, e.* FROM ${EV} e LEFT JOIN dmap m ON e.device_id = m.device_id)`;
|
|
971
|
+
|
|
972
|
+
// Temporal boundaries computed from the same knobs the hooks use
|
|
973
|
+
const MIG_TS = dayjs.utc(DATASET_START).add(MODEL_MIGRATION_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
974
|
+
const OUTAGE_START_TS = dayjs.utc(DATASET_START).add(OUTAGE_START_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
975
|
+
const OUTAGE_END_TS = dayjs.utc(DATASET_START).add(OUTAGE_END_DAY, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
976
|
+
// H4 eligibility: users must have ≥14d of post-week-1 runway, else a natural
|
|
977
|
+
// short tail is indistinguishable from the engineered cliff
|
|
978
|
+
const H4_ELIGIBLE_TS = dayjs.utc(DATASET_END).subtract(21, "day").format("YYYY-MM-DD HH:mm:ss");
|
|
979
|
+
|
|
980
|
+
// H3/H7 cohort cells — EXACTLY the hook's classification. The filters (H4/H8)
|
|
981
|
+
// run before the cohort mutators, so thresholds applied to the output
|
|
982
|
+
// reproduce the hook's cohorts 1:1 (H3's clones only add multi-turn api calls
|
|
983
|
+
// to users already at/above both agentic thresholds; non-members are untouched).
|
|
984
|
+
const CELL_CTE = `coh AS (SELECT e.uid,
|
|
985
|
+
(count(*) FILTER (WHERE e.event = 'tool use call') >= ${AGENTIC_TOOL_THRESHOLD}
|
|
986
|
+
AND count(*) FILTER (WHERE e.event = 'api call' AND e.multi_turn = true) >= ${AGENTIC_MULTITURN_THRESHOLD}) AS agentic,
|
|
987
|
+
bool_or(e.event = 'batch job submitted') AS batch
|
|
988
|
+
FROM ev e GROUP BY 1),
|
|
989
|
+
cells AS (SELECT uid, CASE WHEN agentic AND batch THEN 'both' WHEN agentic THEN 'agentic'
|
|
990
|
+
WHEN batch THEN 'batch' ELSE 'neither' END AS cell FROM coh)`;
|
|
991
|
+
|
|
992
|
+
// Per-user first-event anchor (H4/H8). min(time) in the output equals the
|
|
993
|
+
// hook's anchor because both filters only drop strictly-later events.
|
|
994
|
+
const T0_CTE = `t0 AS (SELECT uid, min(t) AS t0 FROM ev GROUP BY 1)`;
|
|
995
|
+
|
|
996
|
+
// H10 cohorts: docs strictly between earliest org-created and earliest billing
|
|
997
|
+
// payment. Both hook branches are amount-only mutations (nothing dropped, no
|
|
998
|
+
// events injected), so every event the hook classified on survives to the
|
|
999
|
+
// output — the window is exactly rebuildable. The else-bin is 'zero' (docs_ct
|
|
1000
|
+
// = 0, the modal case at ~43% of org∩billing users).
|
|
1001
|
+
const DOCS_CTE = `org AS (SELECT uid, min(t) AS org_t FROM ev WHERE event = 'organization created' GROUP BY 1),
|
|
1002
|
+
bill AS (SELECT uid, min(t) AS bill_t FROM ev WHERE event = 'billing payment' GROUP BY 1),
|
|
1003
|
+
docs AS (SELECT o.uid, count(e.uid) AS docs_ct
|
|
1004
|
+
FROM org o JOIN bill b ON b.uid = o.uid
|
|
1005
|
+
LEFT JOIN ev e ON e.uid = o.uid AND e.event = 'docs searched' AND e.t > o.org_t AND e.t < b.bill_t
|
|
1006
|
+
GROUP BY 1, o.org_t, b.bill_t),
|
|
1007
|
+
dcoh AS (SELECT uid, CASE WHEN docs_ct BETWEEN ${DOCS_SWEET_MIN} AND ${DOCS_SWEET_MAX} THEN 'sweet'
|
|
1008
|
+
WHEN docs_ct >= ${DOCS_OVER_THRESHOLD} THEN 'over' ELSE 'zero' END AS grp FROM docs)`;
|
|
1009
|
+
|
|
1010
|
+
/** @type {import("../../../types").DungeonStory[]} */
|
|
1011
|
+
export const stories = [
|
|
1012
|
+
{
|
|
1013
|
+
id: "H1-prompt-caching",
|
|
1014
|
+
hook: "H1",
|
|
1015
|
+
archetype: "cohort-prop-scale",
|
|
1016
|
+
narrative: `~25% of users (profile distinct_id charcode-sum % ${CACHE_USER_HASH_MOD} === 0) activate prompt caching ~${CACHE_ACTIVATION_PCT * 100}% into their stream; from then on api calls carry cache_enabled=true and cost_usd × ${CACHE_COST_FACTOR}. The flag is stamped, so the breakdown is direct`,
|
|
1017
|
+
assertions: [
|
|
1018
|
+
{
|
|
1019
|
+
breakdown: {
|
|
1020
|
+
type: "duckdb",
|
|
1021
|
+
sql: `SELECT CASE WHEN cache_enabled = true THEN 'cached' ELSE 'uncached' END AS grp,
|
|
1022
|
+
avg(cost_usd) AS avg_cost, count(*) AS event_count
|
|
1023
|
+
FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
|
|
1024
|
+
},
|
|
1025
|
+
select: { c: { where: { grp: "cached" } }, u: { where: { grp: "uncached" } } },
|
|
1026
|
+
// knob 0.3; both sides draw from the same declared cost distribution
|
|
1027
|
+
expect: { metric: "c.avg_cost / u.avg_cost", op: "between", target: [0.24, 0.36] },
|
|
1028
|
+
},
|
|
1029
|
+
{
|
|
1030
|
+
// hash cohort share: charcode-sum % 4 of GUID-ish ids ≈ uniform → ~25%
|
|
1031
|
+
breakdown: {
|
|
1032
|
+
type: "duckdb",
|
|
1033
|
+
sql: `WITH ${ID_CTE},
|
|
1034
|
+
per AS (SELECT uid, bool_or(cache_enabled = true) AS is_cache FROM ev WHERE event = 'api call' GROUP BY 1)
|
|
1035
|
+
SELECT 'all' AS grp, count(*) AS user_count,
|
|
1036
|
+
count(*) FILTER (WHERE is_cache)::DOUBLE / count(*) AS cache_share
|
|
1037
|
+
FROM per`,
|
|
1038
|
+
},
|
|
1039
|
+
select: { all: { where: { grp: "all" } } },
|
|
1040
|
+
expect: { metric: "all.cache_share", op: "between", target: [0.17, 0.33] },
|
|
1041
|
+
},
|
|
1042
|
+
],
|
|
1043
|
+
},
|
|
1044
|
+
{
|
|
1045
|
+
id: "H2-model-migration",
|
|
1046
|
+
hook: "H2",
|
|
1047
|
+
archetype: "temporal-inflection",
|
|
1048
|
+
narrative: `opus-4-7 releases at day ${MODEL_MIGRATION_DAY}: ${MODEL_MIGRATION_LIKELIHOOD}% of post-release Build/Enterprise api calls migrate, at ${MODEL_MIGRATION_TOKEN_MULT}x tokens. The hook scrubs engine-sampled opus-4-7 back to the pre-release mix, so purity is exact: zero opus-4-7 before the release instant, on Free users, or on any non-api-call event`,
|
|
1049
|
+
assertions: [
|
|
1050
|
+
{
|
|
1051
|
+
// deterministic purity — the scrub + tier/date-gated stamp make
|
|
1052
|
+
// any impure row a hook bug, not sampling noise
|
|
1053
|
+
breakdown: {
|
|
1054
|
+
type: "duckdb",
|
|
1055
|
+
sql: `SELECT 'all' AS grp,
|
|
1056
|
+
count(*) FILTER (WHERE model = 'opus-4-7' AND (time::TIMESTAMP < TIMESTAMP '${MIG_TS}' OR api_tier = 'Free' OR event <> 'api call')) AS impure,
|
|
1057
|
+
count(*) FILTER (WHERE model = 'opus-4-7') AS opus_calls
|
|
1058
|
+
FROM ${EV} WHERE model IS NOT NULL`,
|
|
1059
|
+
},
|
|
1060
|
+
assert: (rows) => {
|
|
1061
|
+
const r = (rows || [])[0];
|
|
1062
|
+
if (!r) return { pass: false, verdict: "NONE", detail: "no rows" };
|
|
1063
|
+
if (Number(r.opus_calls) === 0) return { pass: false, verdict: "NONE", detail: "no opus-4-7 calls at all" };
|
|
1064
|
+
const clean = Number(r.impure) === 0;
|
|
1065
|
+
return {
|
|
1066
|
+
pass: clean,
|
|
1067
|
+
verdict: clean ? "NAILED" : "INVERSE",
|
|
1068
|
+
detail: `impure=${r.impure} of ${r.opus_calls} opus-4-7 rows (pre-release / Free / non-api-call must all be 0)`,
|
|
1069
|
+
};
|
|
1070
|
+
},
|
|
1071
|
+
},
|
|
1072
|
+
{
|
|
1073
|
+
// per-call migration is Bernoulli(0.35) — share of paid post-release calls
|
|
1074
|
+
breakdown: {
|
|
1075
|
+
type: "duckdb",
|
|
1076
|
+
sql: `SELECT 'all' AS grp, count(*) AS event_count,
|
|
1077
|
+
count(*) FILTER (WHERE model = 'opus-4-7')::DOUBLE / count(*) AS share
|
|
1078
|
+
FROM ${EV} WHERE event = 'api call' AND api_tier IN ('Build', 'Enterprise')
|
|
1079
|
+
AND time::TIMESTAMP >= TIMESTAMP '${MIG_TS}'`,
|
|
1080
|
+
},
|
|
1081
|
+
select: { all: { where: { grp: "all" } } },
|
|
1082
|
+
expect: { metric: "all.share", op: "between", target: [0.3, 0.4] },
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
// tokens 1.5x — restricted to non-agentic/non-batch users so H3's 8x
|
|
1086
|
+
// and H7's 2x (which hit opus and non-opus calls of their cohorts
|
|
1087
|
+
// alike) can't blur the comparison
|
|
1088
|
+
breakdown: {
|
|
1089
|
+
type: "duckdb",
|
|
1090
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1091
|
+
SELECT CASE WHEN e.model = 'opus-4-7' THEN 'opus' ELSE 'other' END AS grp,
|
|
1092
|
+
avg(e.tokens_used) AS avg_tokens, count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1093
|
+
FROM ev e JOIN cells c ON c.uid = e.uid AND c.cell = 'neither'
|
|
1094
|
+
WHERE e.event = 'api call' AND e.api_tier IN ('Build', 'Enterprise')
|
|
1095
|
+
AND e.t >= TIMESTAMP '${MIG_TS}'
|
|
1096
|
+
GROUP BY 1`,
|
|
1097
|
+
},
|
|
1098
|
+
select: { o: { where: { grp: "opus" } }, x: { where: { grp: "other" } } },
|
|
1099
|
+
expect: { metric: "o.avg_tokens / x.avg_tokens", op: "between", target: [1.3, 1.7] },
|
|
1100
|
+
minCohort: 100,
|
|
1101
|
+
},
|
|
1102
|
+
],
|
|
1103
|
+
},
|
|
1104
|
+
{
|
|
1105
|
+
id: "H3-agentic-power-users",
|
|
1106
|
+
hook: "H3",
|
|
1107
|
+
archetype: "cohort-prop-scale",
|
|
1108
|
+
narrative: `users with ${AGENTIC_TOOL_THRESHOLD}+ tool use calls AND ${AGENTIC_MULTITURN_THRESHOLD}+ multi-turn api calls (classified post-filter — exactly rebuildable) get ${AGENTIC_TOKEN_MULT}x tokens_used and ${AGENTIC_CLONE_MULT} clones per api call. Four-cell design with H7: neither/agentic/batch/both = 1x/${AGENTIC_TOKEN_MULT}x/${BATCH_TOKEN_MULT}x/${AGENTIC_TOKEN_MULT * BATCH_TOKEN_MULT}x. Clones stamp multi_turn=true → agentic multi-turn share ≈ 75% (the 3x-volume signature)`,
|
|
1109
|
+
assertions: [
|
|
1110
|
+
{
|
|
1111
|
+
breakdown: {
|
|
1112
|
+
type: "duckdb",
|
|
1113
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1114
|
+
SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
|
|
1115
|
+
count(*) FILTER (WHERE e.multi_turn = true)::DOUBLE / count(*) AS mt_share,
|
|
1116
|
+
count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1117
|
+
FROM cells c JOIN ev e ON e.uid = c.uid
|
|
1118
|
+
WHERE e.event = 'api call' GROUP BY 1`,
|
|
1119
|
+
},
|
|
1120
|
+
select: { a: { where: { grp: "agentic" } }, n: { where: { grp: "neither" } } },
|
|
1121
|
+
// knob 8x; H2's 1.5x rides both cells (tier ⊥ cohort) and cancels
|
|
1122
|
+
expect: { metric: "a.avg_tokens / n.avg_tokens", op: "between", target: [6.4, 9.6] },
|
|
1123
|
+
minCohort: 50,
|
|
1124
|
+
},
|
|
1125
|
+
{
|
|
1126
|
+
breakdown: {
|
|
1127
|
+
type: "duckdb",
|
|
1128
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1129
|
+
SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
|
|
1130
|
+
count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1131
|
+
FROM cells c JOIN ev e ON e.uid = c.uid
|
|
1132
|
+
WHERE e.event = 'api call' GROUP BY 1`,
|
|
1133
|
+
},
|
|
1134
|
+
select: { b: { where: { grp: "both" } }, n: { where: { grp: "neither" } } },
|
|
1135
|
+
// the deliberate H3×H7 compound: 8 × 2 = 16x
|
|
1136
|
+
expect: { metric: "b.avg_tokens / n.avg_tokens", op: "between", target: [12.8, 19.2] },
|
|
1137
|
+
minCohort: 40,
|
|
1138
|
+
},
|
|
1139
|
+
{
|
|
1140
|
+
breakdown: {
|
|
1141
|
+
type: "duckdb",
|
|
1142
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1143
|
+
SELECT c.cell AS grp,
|
|
1144
|
+
count(*) FILTER (WHERE e.multi_turn = true)::DOUBLE / count(*) AS mt_share,
|
|
1145
|
+
count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1146
|
+
FROM cells c JOIN ev e ON e.uid = c.uid
|
|
1147
|
+
WHERE e.event = 'api call' GROUP BY 1`,
|
|
1148
|
+
},
|
|
1149
|
+
select: { a: { where: { grp: "agentic" } } },
|
|
1150
|
+
// (0.25n + 2n)/3n = 0.75 — clone-volume signature; the declared
|
|
1151
|
+
// multi_turn mix is 1-in-4
|
|
1152
|
+
expect: { metric: "a.mt_share", op: "between", target: [0.62, 0.85] },
|
|
1153
|
+
minCohort: 50,
|
|
1154
|
+
},
|
|
1155
|
+
],
|
|
1156
|
+
},
|
|
1157
|
+
{
|
|
1158
|
+
id: "H4-rate-limit-churn",
|
|
1159
|
+
hook: "H4",
|
|
1160
|
+
archetype: "retention-divergence",
|
|
1161
|
+
narrative: `${RATE_LIMIT_CHURN_LIKELIHOOD}% of users with ${RATE_LIMIT_THRESHOLD}+ rate limit errors in their first 7 days lose ALL post-week-1 events (per-user cliff). The signal is the share of flagged users with zero post-week-1 events, DIFFERENCED against the unflagged share to cancel the natural-quiet baseline — selection-free, unlike volume ratios on a burst-selected cohort. Restricted to users with ≥14d of post-week-1 runway`,
|
|
1162
|
+
assertions: [
|
|
1163
|
+
{
|
|
1164
|
+
breakdown: {
|
|
1165
|
+
type: "duckdb",
|
|
1166
|
+
sql: `WITH ${ID_CTE}, ${T0_CTE},
|
|
1167
|
+
rl AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
|
|
1168
|
+
WHERE e.event = 'rate limit error' AND e.t < t0.t0 + INTERVAL 7 DAY
|
|
1169
|
+
GROUP BY 1 HAVING count(*) >= ${RATE_LIMIT_THRESHOLD}),
|
|
1170
|
+
per AS (SELECT t0.uid, (t0.uid IN (SELECT uid FROM rl)) AS flagged,
|
|
1171
|
+
count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL 7 DAY) AS post_ct
|
|
1172
|
+
FROM t0 JOIN ev e ON e.uid = t0.uid
|
|
1173
|
+
WHERE t0.t0 <= TIMESTAMP '${H4_ELIGIBLE_TS}' GROUP BY 1, 2)
|
|
1174
|
+
SELECT CASE WHEN flagged THEN 'flagged' ELSE 'rest' END AS grp,
|
|
1175
|
+
count(*) AS user_count,
|
|
1176
|
+
count(*) FILTER (WHERE post_ct = 0)::DOUBLE / count(*) AS zero_share
|
|
1177
|
+
FROM per GROUP BY 1`,
|
|
1178
|
+
},
|
|
1179
|
+
select: { f: { where: { grp: "flagged" } }, r: { where: { grp: "rest" } } },
|
|
1180
|
+
// knob 0.6 churn probability; differencing cancels the baseline
|
|
1181
|
+
expect: { metric: "f.zero_share - r.zero_share", op: "between", target: [0.45, 0.7] },
|
|
1182
|
+
minCohort: 50,
|
|
1183
|
+
},
|
|
1184
|
+
{
|
|
1185
|
+
// direct knob readout: flagged zero-post share ≈ 0.6 + tiny baseline
|
|
1186
|
+
breakdown: {
|
|
1187
|
+
type: "duckdb",
|
|
1188
|
+
sql: `WITH ${ID_CTE}, ${T0_CTE},
|
|
1189
|
+
rl AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
|
|
1190
|
+
WHERE e.event = 'rate limit error' AND e.t < t0.t0 + INTERVAL 7 DAY
|
|
1191
|
+
GROUP BY 1 HAVING count(*) >= ${RATE_LIMIT_THRESHOLD}),
|
|
1192
|
+
per AS (SELECT t0.uid,
|
|
1193
|
+
count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL 7 DAY) AS post_ct
|
|
1194
|
+
FROM t0 JOIN ev e ON e.uid = t0.uid
|
|
1195
|
+
WHERE t0.uid IN (SELECT uid FROM rl) AND t0.t0 <= TIMESTAMP '${H4_ELIGIBLE_TS}'
|
|
1196
|
+
GROUP BY 1)
|
|
1197
|
+
SELECT 'flagged' AS grp, count(*) AS user_count,
|
|
1198
|
+
count(*) FILTER (WHERE post_ct = 0)::DOUBLE / count(*) AS zero_share
|
|
1199
|
+
FROM per`,
|
|
1200
|
+
},
|
|
1201
|
+
select: { f: { where: { grp: "flagged" } } },
|
|
1202
|
+
expect: { metric: "f.zero_share", op: "between", target: [0.5, 0.75] },
|
|
1203
|
+
minCohort: 50,
|
|
1204
|
+
},
|
|
1205
|
+
],
|
|
1206
|
+
},
|
|
1207
|
+
{
|
|
1208
|
+
id: "H5-tier-context-window",
|
|
1209
|
+
hook: "H5",
|
|
1210
|
+
archetype: "cohort-prop-scale",
|
|
1211
|
+
narrative: `input_tokens scaled ${TIER_INPUT_MULT.Free}/${TIER_INPUT_MULT.Build}/${TIER_INPUT_MULT.Enterprise}x and context_window pinned to ${TIER_CONTEXT_WINDOW.Free}/${TIER_CONTEXT_WINDOW.Build}/${TIER_CONTEXT_WINDOW.Enterprise} by api_tier. No other hook touches either prop — clean stamp-phase constants`,
|
|
1212
|
+
assertions: [
|
|
1213
|
+
{
|
|
1214
|
+
breakdown: {
|
|
1215
|
+
type: "duckdb",
|
|
1216
|
+
sql: `SELECT api_tier AS grp, avg(input_tokens) AS avg_in, avg(context_window) AS avg_cw,
|
|
1217
|
+
count(*) AS event_count
|
|
1218
|
+
FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
|
|
1219
|
+
},
|
|
1220
|
+
select: { e: { where: { grp: "Enterprise" } }, f: { where: { grp: "Free" } } },
|
|
1221
|
+
// knob 4x (Math.floor truncation is sub-1% at these magnitudes)
|
|
1222
|
+
expect: { metric: "e.avg_in / f.avg_in", op: "between", target: [3.5, 4.5] },
|
|
1223
|
+
},
|
|
1224
|
+
{
|
|
1225
|
+
breakdown: {
|
|
1226
|
+
type: "duckdb",
|
|
1227
|
+
sql: `SELECT api_tier AS grp, avg(context_window) AS avg_cw, min(context_window) AS min_cw,
|
|
1228
|
+
max(context_window) AS max_cw, count(*) AS event_count
|
|
1229
|
+
FROM ${EV} WHERE event = 'api call' GROUP BY 1`,
|
|
1230
|
+
},
|
|
1231
|
+
assert: (rows) => {
|
|
1232
|
+
const want = { Free: TIER_CONTEXT_WINDOW.Free, Build: TIER_CONTEXT_WINDOW.Build, Enterprise: TIER_CONTEXT_WINDOW.Enterprise };
|
|
1233
|
+
const by = Object.fromEntries((rows || []).map((r) => [r.grp, r]));
|
|
1234
|
+
const bad = Object.entries(want).filter(([tier, cw]) =>
|
|
1235
|
+
!by[tier] || Number(by[tier].min_cw) !== cw || Number(by[tier].max_cw) !== cw);
|
|
1236
|
+
const detail = Object.keys(want).map((tr) => `${tr}=${by[tr] ? `${by[tr].min_cw}..${by[tr].max_cw}` : "missing"}`).join(" ");
|
|
1237
|
+
return {
|
|
1238
|
+
pass: bad.length === 0,
|
|
1239
|
+
verdict: bad.length === 0 ? "NAILED" : "INVERSE",
|
|
1240
|
+
detail: `${detail} (every api call must carry its tier's exact constant)`,
|
|
1241
|
+
};
|
|
1242
|
+
},
|
|
1243
|
+
},
|
|
1244
|
+
],
|
|
1245
|
+
},
|
|
1246
|
+
{
|
|
1247
|
+
id: "H6-outage-day",
|
|
1248
|
+
hook: "H6",
|
|
1249
|
+
archetype: "temporal-inflection",
|
|
1250
|
+
narrative: `days ${OUTAGE_START_DAY}-${OUTAGE_END_DAY - 1}: ${OUTAGE_ERROR_LIKELIHOOD}% of api calls flagged is_error with a service error_type and ${OUTAGE_LATENCY_MULT}x latency. is_error declares [false], so the outage is the only error source — the window boundary is exact`,
|
|
1251
|
+
assertions: [
|
|
1252
|
+
{
|
|
1253
|
+
breakdown: {
|
|
1254
|
+
type: "duckdb",
|
|
1255
|
+
sql: `SELECT 'all' AS grp,
|
|
1256
|
+
count(*) FILTER (WHERE is_error = true AND time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}')::DOUBLE
|
|
1257
|
+
/ nullif(count(*) FILTER (WHERE time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}'), 0) AS in_share,
|
|
1258
|
+
count(*) FILTER (WHERE time::TIMESTAMP >= TIMESTAMP '${OUTAGE_START_TS}' AND time::TIMESTAMP < TIMESTAMP '${OUTAGE_END_TS}') AS in_calls,
|
|
1259
|
+
count(*) FILTER (WHERE is_error = true AND (time::TIMESTAMP < TIMESTAMP '${OUTAGE_START_TS}' OR time::TIMESTAMP >= TIMESTAMP '${OUTAGE_END_TS}')) AS out_errors
|
|
1260
|
+
FROM ${EV} WHERE event = 'api call'`,
|
|
1261
|
+
},
|
|
1262
|
+
select: { all: { where: { grp: "all" } } },
|
|
1263
|
+
// knob 40% (Bernoulli per in-window call)
|
|
1264
|
+
expect: { metric: "all.in_share", op: "between", target: [0.35, 0.45] },
|
|
1265
|
+
},
|
|
1266
|
+
{
|
|
1267
|
+
breakdown: {
|
|
1268
|
+
type: "duckdb",
|
|
1269
|
+
sql: `SELECT 'all' AS grp,
|
|
1270
|
+
count(*) FILTER (WHERE is_error = true AND (time::TIMESTAMP < TIMESTAMP '${OUTAGE_START_TS}' OR time::TIMESTAMP >= TIMESTAMP '${OUTAGE_END_TS}')) AS out_errors,
|
|
1271
|
+
count(*) FILTER (WHERE is_error = true) AS total_errors
|
|
1272
|
+
FROM ${EV} WHERE event = 'api call'`,
|
|
1273
|
+
},
|
|
1274
|
+
assert: (rows) => {
|
|
1275
|
+
const r = (rows || [])[0];
|
|
1276
|
+
if (!r) return { pass: false, verdict: "NONE", detail: "no rows" };
|
|
1277
|
+
if (Number(r.total_errors) === 0) return { pass: false, verdict: "NONE", detail: "no errors at all — outage never fired" };
|
|
1278
|
+
const clean = Number(r.out_errors) === 0;
|
|
1279
|
+
return {
|
|
1280
|
+
pass: clean,
|
|
1281
|
+
verdict: clean ? "NAILED" : "INVERSE",
|
|
1282
|
+
detail: `out-of-window errors=${r.out_errors} of ${r.total_errors} total (baseline is 0% by schema)`,
|
|
1283
|
+
};
|
|
1284
|
+
},
|
|
1285
|
+
},
|
|
1286
|
+
],
|
|
1287
|
+
},
|
|
1288
|
+
{
|
|
1289
|
+
id: "H7-batch-discount",
|
|
1290
|
+
hook: "H7",
|
|
1291
|
+
archetype: "cohort-prop-scale",
|
|
1292
|
+
narrative: `users with any surviving batch job submitted get cost_per_token × ${BATCH_COST_FACTOR} (touched by no other hook) and tokens_used × ${BATCH_TOKEN_MULT} (compounds with H3 — the 'both' cell is verified in the H3 story)`,
|
|
1293
|
+
assertions: [
|
|
1294
|
+
{
|
|
1295
|
+
breakdown: {
|
|
1296
|
+
type: "duckdb",
|
|
1297
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1298
|
+
SELECT CASE WHEN c.cell IN ('batch', 'both') THEN 'batch' ELSE 'rest' END AS grp,
|
|
1299
|
+
avg(e.cost_per_token) AS avg_cpt, count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1300
|
+
FROM cells c JOIN ev e ON e.uid = c.uid
|
|
1301
|
+
WHERE e.event = 'api call' GROUP BY 1`,
|
|
1302
|
+
},
|
|
1303
|
+
select: { b: { where: { grp: "batch" } }, r: { where: { grp: "rest" } } },
|
|
1304
|
+
// knob 0.5; cost_per_token has no other mutator
|
|
1305
|
+
expect: { metric: "b.avg_cpt / r.avg_cpt", op: "between", target: [0.42, 0.58] },
|
|
1306
|
+
minCohort: 100,
|
|
1307
|
+
},
|
|
1308
|
+
{
|
|
1309
|
+
breakdown: {
|
|
1310
|
+
type: "duckdb",
|
|
1311
|
+
sql: `WITH ${ID_CTE}, ${CELL_CTE}
|
|
1312
|
+
SELECT c.cell AS grp, avg(e.tokens_used) AS avg_tokens,
|
|
1313
|
+
count(*) AS event_count, count(DISTINCT e.uid) AS user_count
|
|
1314
|
+
FROM cells c JOIN ev e ON e.uid = c.uid
|
|
1315
|
+
WHERE e.event = 'api call' GROUP BY 1`,
|
|
1316
|
+
},
|
|
1317
|
+
select: { b: { where: { grp: "batch" } }, n: { where: { grp: "neither" } } },
|
|
1318
|
+
// batch-only cell: clean 2x (agentic users are in their own cells)
|
|
1319
|
+
expect: { metric: "b.avg_tokens / n.avg_tokens", op: "between", target: [1.7, 2.3] },
|
|
1320
|
+
minCohort: 100,
|
|
1321
|
+
},
|
|
1322
|
+
],
|
|
1323
|
+
},
|
|
1324
|
+
{
|
|
1325
|
+
id: "H8-eval-retention",
|
|
1326
|
+
hook: "H8",
|
|
1327
|
+
archetype: "retention-divergence",
|
|
1328
|
+
narrative: `users without an eval job in their first 7 days keep only ${EVAL_NON_USER_KEEP_LIKELIHOOD}% of post-day-${EVAL_CUTOFF_DAYS} events. Ratio-of-ratios (noneval post/pre vs eval post/pre) cancels window lengths and the growth soup; H4's independent drop rides both cohorts`,
|
|
1329
|
+
assertions: [
|
|
1330
|
+
{
|
|
1331
|
+
breakdown: {
|
|
1332
|
+
type: "duckdb",
|
|
1333
|
+
sql: `WITH ${ID_CTE}, ${T0_CTE},
|
|
1334
|
+
ev_users AS (SELECT e.uid FROM ev e JOIN t0 ON t0.uid = e.uid
|
|
1335
|
+
WHERE e.event = 'eval job' AND e.t < t0.t0 + INTERVAL 7 DAY GROUP BY 1),
|
|
1336
|
+
per AS (SELECT t0.uid, (t0.uid IN (SELECT uid FROM ev_users)) AS eval_user,
|
|
1337
|
+
count(*) FILTER (WHERE e.t <= t0.t0 + INTERVAL ${EVAL_CUTOFF_DAYS} DAY) AS pre_ct,
|
|
1338
|
+
count(*) FILTER (WHERE e.t > t0.t0 + INTERVAL ${EVAL_CUTOFF_DAYS} DAY) AS post_ct
|
|
1339
|
+
FROM t0 JOIN ev e ON e.uid = t0.uid GROUP BY 1, 2)
|
|
1340
|
+
SELECT CASE WHEN eval_user THEN 'eval' ELSE 'noneval' END AS grp,
|
|
1341
|
+
count(*) AS user_count, avg(post_ct) AS avg_post, avg(pre_ct) AS avg_pre,
|
|
1342
|
+
avg(post_ct) / nullif(avg(pre_ct), 0) AS post_pre
|
|
1343
|
+
FROM per GROUP BY 1`,
|
|
1344
|
+
},
|
|
1345
|
+
select: { n: { where: { grp: "noneval" } }, e: { where: { grp: "eval" } } },
|
|
1346
|
+
// knob keep-rate 0.25
|
|
1347
|
+
expect: { metric: "n.post_pre / e.post_pre", op: "between", target: [0.17, 0.34] },
|
|
1348
|
+
minCohort: 50,
|
|
1349
|
+
},
|
|
1350
|
+
],
|
|
1351
|
+
},
|
|
1352
|
+
{
|
|
1353
|
+
id: "H9-api-to-eval-ttc",
|
|
1354
|
+
hook: "H9",
|
|
1355
|
+
archetype: "funnel-ttc-by-segment",
|
|
1356
|
+
narrative: `funnel-post scales API-to-Eval step gaps by tier: Enterprise × ${FUNNEL_TTC_ENTERPRISE}, Free × ${FUNNEL_TTC_FREE}, Build 1x. Measured with the Mixpanel-aligned funnel emulator (greedy step pairing), NOT raw SQL: nearest-preceding-pair SQL is censored by the fixed lookback window — stretching Free gaps pushes true pairs past the window edge and intercepts more organic events, which INVERTS the measured direction. The emulator window is 336h = max scale factor (${FUNNEL_TTC_FREE}) × the funnel's 168h generative window, so the stretched support fits — at 168h any Free instance whose original TTC exceeded 84h fails the window and the longest (most-stretched) pairs censor out. Greedy pairing still dilutes toward 1 (organic same-window events get picked as steps), asymmetrically — stretch (Free) dilutes harder than compress (Enterprise). Bands reflect the diluted effect, not the pure knobs`,
|
|
1357
|
+
assertions: [
|
|
1358
|
+
{
|
|
1359
|
+
breakdown: {
|
|
1360
|
+
type: "timeToConvert",
|
|
1361
|
+
steps: ["api call", "tool use call", "eval job"],
|
|
1362
|
+
breakdownByUserProperty: "api_tier",
|
|
1363
|
+
// 336h = FUNNEL_TTC_FREE × generative 168h window (covers stretched support)
|
|
1364
|
+
conversionWindowMs: 336 * 60 * 60 * 1000,
|
|
1365
|
+
},
|
|
1366
|
+
select: { e: { where: { segment_value: "Enterprise" } }, b: { where: { segment_value: "Build" } } },
|
|
1367
|
+
// knob 0.5 pure; greedy-pairing dilution pulls toward 1
|
|
1368
|
+
expect: { metric: "e.median_ttc_ms / b.median_ttc_ms", op: "between", target: [0.55, 0.92] },
|
|
1369
|
+
minCohort: 300,
|
|
1370
|
+
},
|
|
1371
|
+
{
|
|
1372
|
+
breakdown: {
|
|
1373
|
+
type: "timeToConvert",
|
|
1374
|
+
steps: ["api call", "tool use call", "eval job"],
|
|
1375
|
+
breakdownByUserProperty: "api_tier",
|
|
1376
|
+
// 336h = FUNNEL_TTC_FREE × generative 168h window (covers stretched support)
|
|
1377
|
+
conversionWindowMs: 336 * 60 * 60 * 1000,
|
|
1378
|
+
},
|
|
1379
|
+
select: { f: { where: { segment_value: "Free" } }, b: { where: { segment_value: "Build" } } },
|
|
1380
|
+
// knob 2.0 pure; stretch dilutes harder than compress
|
|
1381
|
+
expect: { metric: "f.median_ttc_ms / b.median_ttc_ms", op: "between", target: [1.1, 2.2] },
|
|
1382
|
+
minCohort: 300,
|
|
1383
|
+
},
|
|
1384
|
+
],
|
|
1385
|
+
},
|
|
1386
|
+
{
|
|
1387
|
+
id: "H10-docs-magic-number",
|
|
1388
|
+
hook: "H10",
|
|
1389
|
+
archetype: "frequency-sweet-spot",
|
|
1390
|
+
narrative: `docs searched between org-created and first billing: ${DOCS_SWEET_MIN}-${DOCS_SWEET_MAX} (sweet) → amount_usd × ${DOCS_BILLING_BOOST} on all billing payments; ${DOCS_OVER_THRESHOLD}+ (over) → amount_usd × ${DOCS_OVER_PENALTY} on all billing payments. Both branches mutate an iid-drawn property — selection-free, unlike count effects which drown in activity-selection bias at ~100-user cohorts (placebo on an untouched cohort read 1.22-1.30 under the best normalizer). Median ratios against the untouched zero-docs cohort read the knobs directly (amount_usd draw is docs-count-independent; ×k is monotone so median scales by k; Math.round is sub-1% at these medians)`,
|
|
1391
|
+
assertions: [
|
|
1392
|
+
{
|
|
1393
|
+
breakdown: {
|
|
1394
|
+
type: "duckdb",
|
|
1395
|
+
sql: `WITH ${ID_CTE}, ${DOCS_CTE}
|
|
1396
|
+
SELECT d.grp AS grp, median(e.amount_usd) AS med_amt, count(*) AS payment_count,
|
|
1397
|
+
count(DISTINCT d.uid) AS user_count
|
|
1398
|
+
FROM dcoh d JOIN ev e ON e.uid = d.uid AND e.event = 'billing payment'
|
|
1399
|
+
GROUP BY 1`,
|
|
1400
|
+
},
|
|
1401
|
+
select: { s: { where: { grp: "sweet" } }, z: { where: { grp: "zero" } } },
|
|
1402
|
+
// knob 1.35 ±15%
|
|
1403
|
+
expect: { metric: "s.med_amt / z.med_amt", op: "between", target: [1.15, 1.55] },
|
|
1404
|
+
minCohort: 60,
|
|
1405
|
+
},
|
|
1406
|
+
{
|
|
1407
|
+
breakdown: {
|
|
1408
|
+
type: "duckdb",
|
|
1409
|
+
sql: `WITH ${ID_CTE}, ${DOCS_CTE}
|
|
1410
|
+
SELECT d.grp AS grp, median(e.amount_usd) AS med_amt, count(*) AS payment_count,
|
|
1411
|
+
count(DISTINCT d.uid) AS user_count
|
|
1412
|
+
FROM dcoh d JOIN ev e ON e.uid = d.uid AND e.event = 'billing payment'
|
|
1413
|
+
GROUP BY 1`,
|
|
1414
|
+
},
|
|
1415
|
+
select: { o: { where: { grp: "over" } }, z: { where: { grp: "zero" } } },
|
|
1416
|
+
// knob 0.75 ±15%
|
|
1417
|
+
expect: { metric: "o.med_amt / z.med_amt", op: "between", target: [0.64, 0.86] },
|
|
1418
|
+
minCohort: 60,
|
|
1419
|
+
},
|
|
1420
|
+
],
|
|
1421
|
+
},
|
|
1422
|
+
];
|
|
1423
|
+
|
|
1424
|
+
export default config;
|