@ak--47/dungeon-master 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +51 -13
- package/dungeons/technical/ad-spend.js +2 -2
- package/dungeons/technical/anonymous-users.js +2 -2
- package/dungeons/technical/array-of-object-lookup.js +2 -4
- package/dungeons/technical/experiments.js +2 -2
- package/dungeons/technical/foobar.js +2 -2
- package/dungeons/technical/group-analytics.js +2 -2
- package/dungeons/technical/mirror-strategies.js +2 -2
- package/dungeons/technical/nested-objects.js +2 -2
- package/dungeons/technical/retention-cadence.js +2 -3
- package/dungeons/technical/sanity.js +2 -2
- package/dungeons/technical/scale-test.js +2 -2
- package/dungeons/technical/scd.js +2 -2
- package/dungeons/technical/simple.js +5 -6
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/simplest.js +2 -2
- package/dungeons/technical/text-generation.js +3 -3
- package/dungeons/vertical/ai-platform.js +858 -0
- package/dungeons/vertical/community.js +84 -40
- package/dungeons/vertical/crypto.js +830 -0
- package/dungeons/vertical/dating.js +744 -0
- package/dungeons/vertical/devtools.js +175 -69
- package/dungeons/vertical/ecommerce.js +242 -94
- package/dungeons/vertical/education.js +330 -313
- package/dungeons/vertical/fintech.js +442 -313
- package/dungeons/vertical/fitness.js +143 -61
- package/dungeons/vertical/food-delivery.js +327 -353
- package/dungeons/vertical/gaming.js +912 -382
- package/dungeons/vertical/healthcare.js +142 -63
- package/dungeons/vertical/insurance-application.js +170 -76
- package/dungeons/vertical/logistics.js +115 -20
- package/dungeons/vertical/marketplace.js +152 -58
- package/dungeons/vertical/media.js +248 -384
- package/dungeons/vertical/real-estate.js +781 -0
- package/dungeons/vertical/sass.js +255 -266
- package/dungeons/vertical/social.js +264 -206
- package/dungeons/vertical/travel.js +117 -41
- package/index.js +17 -17
- package/lib/core/config-validator.js +159 -31
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +11 -14
- package/lib/generators/funnels.js +12 -4
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/orchestrators/user-loop.js +81 -47
- package/lib/templates/macro-presets.js +111 -0
- package/lib/templates/soup-presets.js +19 -36
- package/lib/utils/utils.js +71 -39
- package/package.json +8 -2
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +251 -51
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/user/.gitkeep +0 -0
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -438
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/rpg-schema.json +0 -2491
- package/dungeons/vertical/rpg.js +0 -976
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -0,0 +1,858 @@
|
|
|
1
|
+
// ── TWEAK THESE ──
|
|
2
|
+
const SEED = "promptforge";
|
|
3
|
+
const num_days = 120;
|
|
4
|
+
const num_users = 8_000;
|
|
5
|
+
const avg_events_per_user_per_day = 0.83;
|
|
6
|
+
let token = "your-mixpanel-token";
|
|
7
|
+
|
|
8
|
+
// ── env overrides ──
|
|
9
|
+
if (process.env.MP_TOKEN) token = process.env.MP_TOKEN;
|
|
10
|
+
|
|
11
|
+
import dayjs from "dayjs";
|
|
12
|
+
import utc from "dayjs/plugin/utc.js";
|
|
13
|
+
import "dotenv/config";
|
|
14
|
+
import * as u from "../../lib/utils/utils.js";
|
|
15
|
+
import * as v from "ak-tools";
|
|
16
|
+
|
|
17
|
+
dayjs.extend(utc);
|
|
18
|
+
const chance = u.initChance(SEED);
|
|
19
|
+
/** @typedef {import("../../types").Dungeon} Config */
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* ===============================================================
|
|
23
|
+
* DATASET OVERVIEW
|
|
24
|
+
* ===============================================================
|
|
25
|
+
*
|
|
26
|
+
* PromptForge -- an LLM API platform (like Anthropic/OpenAI).
|
|
27
|
+
* Customers (developers and companies) send API requests for chat
|
|
28
|
+
* completions, embeddings, evaluations, and tool use. Billing is
|
|
29
|
+
* per input/output token. Key features: prompt caching, tool use,
|
|
30
|
+
* multi-turn conversations, batch API, model selection, and
|
|
31
|
+
* evaluation pipelines.
|
|
32
|
+
*
|
|
33
|
+
* - 8,000 users over 120 days, ~800K events
|
|
34
|
+
* - Three API tiers: Free, Build, Enterprise
|
|
35
|
+
* - Core loop: org created -> api key created -> api call -> iterate
|
|
36
|
+
* - Revenue: token-based billing with tier-based pricing
|
|
37
|
+
*
|
|
38
|
+
* Key entities:
|
|
39
|
+
* - model: LLM model version (sonnet-4, haiku-4, opus-4-6, opus-4-7)
|
|
40
|
+
* - api_tier: Free / Build / Enterprise (determines context window, rate limits)
|
|
41
|
+
* - tokens_used: total tokens consumed per API call (input + output)
|
|
42
|
+
* - cost_usd: dollar cost of a single API call
|
|
43
|
+
* - cache_enabled: prompt caching flag that reduces cost 70%
|
|
44
|
+
* - multi_turn: whether the call is part of a conversation
|
|
45
|
+
*
|
|
46
|
+
* ===============================================================
|
|
47
|
+
* ANALYTICS HOOKS (10 hooks)
|
|
48
|
+
* ===============================================================
|
|
49
|
+
*
|
|
50
|
+
* NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
51
|
+
* only via behavioral cohorts (count event per user) or raw-prop
|
|
52
|
+
* breakdowns (api_tier from profile, model, error_type).
|
|
53
|
+
*
|
|
54
|
+
* ---------------------------------------------------------------
|
|
55
|
+
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
56
|
+
* ---------------------------------------------------------------
|
|
57
|
+
*
|
|
58
|
+
* PATTERN: Customers who enable prompt caching see 70% lower
|
|
59
|
+
* cost_per_call. Once any api call has cache_enabled=true, all
|
|
60
|
+
* subsequent calls for that user get cost_usd reduced by 70%.
|
|
61
|
+
*
|
|
62
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
63
|
+
*
|
|
64
|
+
* Report 1: Cost Per Call by Cache Status
|
|
65
|
+
* - Report type: Insights
|
|
66
|
+
* - Event: "api call"
|
|
67
|
+
* - Measure: Average of "cost_usd"
|
|
68
|
+
* - Breakdown: "cache_enabled"
|
|
69
|
+
* - Expected: cache_enabled=true ~ $0.003, false ~ $0.01 (70% cheaper)
|
|
70
|
+
*
|
|
71
|
+
* Report 2: Cache Adoption Over Time
|
|
72
|
+
* - Report type: Insights
|
|
73
|
+
* - Event: "api call"
|
|
74
|
+
* - Measure: Total
|
|
75
|
+
* - Filter: cache_enabled = true
|
|
76
|
+
* - Line chart by week
|
|
77
|
+
* - Expected: steady growth in cached calls over the dataset
|
|
78
|
+
*
|
|
79
|
+
* REAL-WORLD ANALOGUE: Prompt caching avoids re-processing long
|
|
80
|
+
* system prompts on every call, dramatically reducing cost and latency.
|
|
81
|
+
*
|
|
82
|
+
* ---------------------------------------------------------------
|
|
83
|
+
* 2. MODEL MIGRATION WAVE (TIMED RELEASE — event)
|
|
84
|
+
* ---------------------------------------------------------------
|
|
85
|
+
*
|
|
86
|
+
* PATTERN: At day 60, new model "opus-4-7" releases. After day 60,
|
|
87
|
+
* 35% of api calls from Build/Enterprise users switch model to
|
|
88
|
+
* "opus-4-7". These calls use 1.5x tokens (smarter model, longer
|
|
89
|
+
* responses).
|
|
90
|
+
*
|
|
91
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
92
|
+
*
|
|
93
|
+
* Report 1: Model Distribution Over Time
|
|
94
|
+
* - Report type: Insights
|
|
95
|
+
* - Event: "api call"
|
|
96
|
+
* - Measure: Total
|
|
97
|
+
* - Breakdown: "model"
|
|
98
|
+
* - Line chart by week
|
|
99
|
+
* - Expected: opus-4-7 appears at day 60, ramps to ~35% of paid calls
|
|
100
|
+
*
|
|
101
|
+
* Report 2: Tokens Per Model
|
|
102
|
+
* - Report type: Insights
|
|
103
|
+
* - Event: "api call"
|
|
104
|
+
* - Measure: Average of "tokens_used"
|
|
105
|
+
* - Breakdown: "model"
|
|
106
|
+
* - Expected: opus-4-7 ~ 1.5x tokens vs other models
|
|
107
|
+
*
|
|
108
|
+
* REAL-WORLD ANALOGUE: New flagship model launches cause migration
|
|
109
|
+
* waves among power users who want improved capabilities.
|
|
110
|
+
*
|
|
111
|
+
* ---------------------------------------------------------------
|
|
112
|
+
* 3. AGENTIC LOOP POWER USERS (everything)
|
|
113
|
+
* ---------------------------------------------------------------
|
|
114
|
+
*
|
|
115
|
+
* PATTERN: Users with both "tool use call" AND any api-call event with
|
|
116
|
+
* multi_turn=true get 8x tokens_used on api calls plus 2 extra cloned
|
|
117
|
+
* api-call events per existing (3x rate). Cloned events with unique
|
|
118
|
+
* offset timestamps. No flag — discover via cohort builder.
|
|
119
|
+
*
|
|
120
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
121
|
+
*
|
|
122
|
+
* Report 1: Tokens per User — Agentic Cohort
|
|
123
|
+
* - Report type: Insights (with cohort)
|
|
124
|
+
* - Cohort A: users with both >= 1 "tool use call" AND >= 1 api-call with multi_turn=true
|
|
125
|
+
* - Cohort B: rest
|
|
126
|
+
* - Event: "api call"
|
|
127
|
+
* - Measure: Average of "tokens_used"
|
|
128
|
+
* - Expected: A ~ 8x B
|
|
129
|
+
*
|
|
130
|
+
* REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
|
|
131
|
+
* tokens via extended tool-use loops.
|
|
132
|
+
*
|
|
133
|
+
* ---------------------------------------------------------------
|
|
134
|
+
* 4. RATE LIMIT CHURN (everything)
|
|
135
|
+
* ---------------------------------------------------------------
|
|
136
|
+
*
|
|
137
|
+
* PATTERN: Users with >= 2 "rate limit error" events in first 7 days
|
|
138
|
+
* lose 60% of events after week 1. No flag — discover via cohort.
|
|
139
|
+
*
|
|
140
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
141
|
+
*
|
|
142
|
+
* Report 1: Retention by Early Rate-Limit Cohort
|
|
143
|
+
* - Report type: Retention
|
|
144
|
+
* - Cohort A: users with >= 2 "rate limit error" in first 7 days
|
|
145
|
+
* - Cohort B: rest
|
|
146
|
+
* - Expected: A retention ~ 40% vs B ~ 80%
|
|
147
|
+
*
|
|
148
|
+
* REAL-WORLD ANALOGUE: Developers who get rate-limited early often
|
|
149
|
+
* switch to a competitor.
|
|
150
|
+
*
|
|
151
|
+
* ---------------------------------------------------------------
|
|
152
|
+
* 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
|
|
153
|
+
* ---------------------------------------------------------------
|
|
154
|
+
*
|
|
155
|
+
* PATTERN: Free users have context_window=200000, Build=1000000,
|
|
156
|
+
* Enterprise=2000000. Enterprise users send 4x larger input_tokens.
|
|
157
|
+
* Context window and input tokens are scaled by tier.
|
|
158
|
+
*
|
|
159
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
160
|
+
*
|
|
161
|
+
* Report 1: Input Tokens by Tier
|
|
162
|
+
* - Report type: Insights
|
|
163
|
+
* - Event: "api call"
|
|
164
|
+
* - Measure: Average of "input_tokens"
|
|
165
|
+
* - Breakdown: "api_tier" (superProp)
|
|
166
|
+
* - Expected: Enterprise ~ 4x Free (Enterprise ~ 8K, Free ~ 2K)
|
|
167
|
+
*
|
|
168
|
+
* Report 2: Context Window by Tier
|
|
169
|
+
* - Report type: Insights
|
|
170
|
+
* - Event: "api call"
|
|
171
|
+
* - Measure: Average of "context_window"
|
|
172
|
+
* - Breakdown: "api_tier"
|
|
173
|
+
* - Expected: Free=200K, Build=1M, Enterprise=2M
|
|
174
|
+
*
|
|
175
|
+
* REAL-WORLD ANALOGUE: Enterprise customers pay for larger context
|
|
176
|
+
* windows and use them for long-document analysis and code review.
|
|
177
|
+
*
|
|
178
|
+
* ---------------------------------------------------------------
|
|
179
|
+
* 6. OUTAGE DAY (TIME-BASED — event)
|
|
180
|
+
* ---------------------------------------------------------------
|
|
181
|
+
*
|
|
182
|
+
* PATTERN: Days 40-41, is_error is set to true on 40% of api call
|
|
183
|
+
* events. error_type is set to service errors. Simulates a major
|
|
184
|
+
* platform outage.
|
|
185
|
+
*
|
|
186
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
187
|
+
*
|
|
188
|
+
* Report 1: Error Rate Over Time
|
|
189
|
+
* - Report type: Insights
|
|
190
|
+
* - Event: "api call"
|
|
191
|
+
* - Measure: Total
|
|
192
|
+
* - Filter: is_error = true
|
|
193
|
+
* - Line chart by day
|
|
194
|
+
* - Expected: massive spike on days 40-41 (8x baseline error rate)
|
|
195
|
+
*
|
|
196
|
+
* Report 2: Error Types During Outage
|
|
197
|
+
* - Report type: Insights
|
|
198
|
+
* - Event: "api call"
|
|
199
|
+
* - Filter: is_error = true
|
|
200
|
+
* - Breakdown: "error_type"
|
|
201
|
+
* - Date range: days 40-41
|
|
202
|
+
* - Expected: service_overloaded and internal_server_error dominate
|
|
203
|
+
*
|
|
204
|
+
* REAL-WORLD ANALOGUE: API platforms experience periodic outages
|
|
205
|
+
* that spike error rates across all customers.
|
|
206
|
+
*
|
|
207
|
+
* ---------------------------------------------------------------
|
|
208
|
+
* 7. BATCH API DISCOUNT (everything)
|
|
209
|
+
* ---------------------------------------------------------------
|
|
210
|
+
*
|
|
211
|
+
* PATTERN: Users with any "batch job submitted" event get 50% lower
|
|
212
|
+
* cost_per_token on api calls + 2x tokens_used. Mutates raw props.
|
|
213
|
+
* No flag — discover via cohort builder.
|
|
214
|
+
*
|
|
215
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
216
|
+
*
|
|
217
|
+
* Report 1: Cost per Token by Batch Cohort
|
|
218
|
+
* - Report type: Insights (with cohort)
|
|
219
|
+
* - Cohort A: users with >= 1 "batch job submitted"
|
|
220
|
+
* - Cohort B: rest
|
|
221
|
+
* - Event: "api call"
|
|
222
|
+
* - Measure: Average of "cost_per_token"
|
|
223
|
+
* - Expected: A ~ 0.5x B
|
|
224
|
+
*
|
|
225
|
+
* REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
|
|
226
|
+
*
|
|
227
|
+
* ---------------------------------------------------------------
|
|
228
|
+
* 8. EVAL-DRIVEN RETENTION (everything)
|
|
229
|
+
* ---------------------------------------------------------------
|
|
230
|
+
*
|
|
231
|
+
* PATTERN: Users with any "eval job" in first 7 days keep all events.
|
|
232
|
+
* Non-eval users lose 75% of post-day-30 events. No flag — discover
|
|
233
|
+
* via retention cohort.
|
|
234
|
+
*
|
|
235
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
236
|
+
*
|
|
237
|
+
* Report 1: Retention by Early Eval Cohort
|
|
238
|
+
* - Report type: Retention
|
|
239
|
+
* - Cohort A: users with >= 1 "eval job" in first 7 days
|
|
240
|
+
* - Cohort B: rest
|
|
241
|
+
* - Expected: A ~ 75% D30 vs B ~ 25%
|
|
242
|
+
*
|
|
243
|
+
* REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
|
|
244
|
+
*
|
|
245
|
+
* ---------------------------------------------------------------
|
|
246
|
+
* 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
247
|
+
* ---------------------------------------------------------------
|
|
248
|
+
*
|
|
249
|
+
* PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
|
|
250
|
+
* 1.5x faster than baseline (factor 0.67 on inter-event gaps); Free
|
|
251
|
+
* users 1.4x slower (factor 1.4). Mutates funnel event timestamps.
|
|
252
|
+
*
|
|
253
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
254
|
+
*
|
|
255
|
+
* Report 1: API to Eval — Median Time-to-Convert by Tier
|
|
256
|
+
* - Report type: Funnels
|
|
257
|
+
* - Steps: "api call" -> "tool use call" -> "eval job"
|
|
258
|
+
* - Measure: Median time to convert
|
|
259
|
+
* - Breakdown: "api_tier"
|
|
260
|
+
* - Expected: Enterprise ~ 0.67x Build; Free ~ 1.4x Build
|
|
261
|
+
*
|
|
262
|
+
* NOTE (funnel-post measurement): visible only via Mixpanel funnel
|
|
263
|
+
* median TTC. Cross-event MIN→MIN SQL queries on raw events do NOT
|
|
264
|
+
* show this — funnel-post adjusts gaps within funnel instances, not
|
|
265
|
+
* across the user's full event history.
|
|
266
|
+
*
|
|
267
|
+
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
|
|
268
|
+
* who execute end-to-end pipelines faster.
|
|
269
|
+
*
|
|
270
|
+
* ---------------------------------------------------------------
|
|
271
|
+
* 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
|
|
272
|
+
* ---------------------------------------------------------------
|
|
273
|
+
*
|
|
274
|
+
* PATTERN: Count "docs searched" events between organization-created and
|
|
275
|
+
* first billing-payment. Sweet 5-8 → +35% on amount_usd of billing
|
|
276
|
+
* payment events. Over 9+ → drop 30% of billing payment events. No flag.
|
|
277
|
+
*
|
|
278
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
279
|
+
*
|
|
280
|
+
* Report 1: Avg Billing Amount by Docs-Searched Bucket
|
|
281
|
+
* - Report type: Insights (with cohort)
|
|
282
|
+
* - Cohort A: users with 5-8 "docs searched" between sign-up and first billing
|
|
283
|
+
* - Cohort B: users with 0-4
|
|
284
|
+
* - Event: "billing payment"
|
|
285
|
+
* - Measure: Average of "amount_usd"
|
|
286
|
+
* - Expected: A ~ 1.35x B
|
|
287
|
+
*
|
|
288
|
+
* Report 2: Billing Payments per User on Heavy Searchers
|
|
289
|
+
* - Report type: Insights (with cohort)
|
|
290
|
+
* - Cohort C: users with >= 9 "docs searched" between sign-up and billing
|
|
291
|
+
* - Cohort A: users with 5-8
|
|
292
|
+
* - Event: "billing payment"
|
|
293
|
+
* - Measure: Total per user
|
|
294
|
+
* - Expected: C ~ 30% fewer billing payments per user
|
|
295
|
+
*
|
|
296
|
+
* REAL-WORLD ANALOGUE: Reading the docs lifts willingness to pay; doc
|
|
297
|
+
* obsession signals stuck on integration and never paying.
|
|
298
|
+
*
|
|
299
|
+
* ===============================================================
|
|
300
|
+
* EXPECTED METRICS SUMMARY
|
|
301
|
+
* ===============================================================
|
|
302
|
+
*
|
|
303
|
+
* Hook | Metric | Baseline | Effect | Ratio
|
|
304
|
+
* ----------------------------|----------------------|------------|--------------|------
|
|
305
|
+
* Prompt Caching Adoption | cost_usd | $0.01 | $0.003 | 0.3x
|
|
306
|
+
* Model Migration Wave | opus-4-7 share paid | 0% | ~ 35% | new
|
|
307
|
+
* Agentic Loop Power Users | tokens/api-call | 1x | 8x | 8x
|
|
308
|
+
* Rate Limit Churn | D30 retention | 80% | 40% | 0.5x
|
|
309
|
+
* Tier-Based Context Window | input_tokens | 2K (Free) | 8K (Ent) | 4x
|
|
310
|
+
* Outage Day | error rate days 40-41| 5% | 40% | 8x
|
|
311
|
+
* Batch API Discount | cost_per_token | 1x | 0.5x | -50%
|
|
312
|
+
* Eval-Driven Retention | D30 retention | 25% | 75% | 3x
|
|
313
|
+
* API-to-Eval T2C | median min by tier | 1x (Build) | 0.67x / 1.4x | 1.5x range
|
|
314
|
+
* Docs Magic Number | sweet billing amount | 1x | 1.35x | 1.35x
|
|
315
|
+
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
316
|
+
*/
|
|
317
|
+
|
|
318
|
+
/** @type {Config} */
|
|
319
|
+
const config = {
|
|
320
|
+
token,
|
|
321
|
+
seed: SEED,
|
|
322
|
+
datasetStart: "2026-01-01T00:00:00Z",
|
|
323
|
+
datasetEnd: "2026-04-28T23:59:59Z",
|
|
324
|
+
// numDays: num_days,
|
|
325
|
+
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
326
|
+
numUsers: num_users,
|
|
327
|
+
hasAnonIds: false,
|
|
328
|
+
hasSessionIds: false,
|
|
329
|
+
format: "json",
|
|
330
|
+
gzip: true,
|
|
331
|
+
alsoInferFunnels: false,
|
|
332
|
+
hasLocation: true,
|
|
333
|
+
hasAndroidDevices: false,
|
|
334
|
+
hasIOSDevices: false,
|
|
335
|
+
hasDesktopDevices: true,
|
|
336
|
+
hasBrowser: true,
|
|
337
|
+
hasCampaigns: false,
|
|
338
|
+
isAnonymous: false,
|
|
339
|
+
hasAdSpend: false,
|
|
340
|
+
hasAvatar: true,
|
|
341
|
+
concurrency: 1,
|
|
342
|
+
writeToDisk: false,
|
|
343
|
+
|
|
344
|
+
soup: "growth",
|
|
345
|
+
|
|
346
|
+
scdProps: {
|
|
347
|
+
monthly_api_usage: {
|
|
348
|
+
values: u.weighNumRange(0, 1000000, 0.3, 50),
|
|
349
|
+
frequency: "week",
|
|
350
|
+
timing: "fuzzy",
|
|
351
|
+
max: 20,
|
|
352
|
+
},
|
|
353
|
+
api_tier_history: {
|
|
354
|
+
values: ["Free", "Build", "Enterprise"],
|
|
355
|
+
frequency: "month",
|
|
356
|
+
timing: "fixed",
|
|
357
|
+
max: 6,
|
|
358
|
+
},
|
|
359
|
+
},
|
|
360
|
+
|
|
361
|
+
// -- Events (18) ------------------------------------------
|
|
362
|
+
events: [
|
|
363
|
+
{
|
|
364
|
+
event: "organization created",
|
|
365
|
+
weight: 1,
|
|
366
|
+
isFirstEvent: true,
|
|
367
|
+
properties: {
|
|
368
|
+
org_size: ["solo", "startup", "growth", "enterprise"],
|
|
369
|
+
referral_source: ["docs", "blog", "github", "word_of_mouth", "search", "conference"],
|
|
370
|
+
},
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
event: "api key created",
|
|
374
|
+
weight: 2,
|
|
375
|
+
properties: {
|
|
376
|
+
key_type: ["development", "production", "staging"],
|
|
377
|
+
key_scope: ["full_access", "read_only", "completions_only"],
|
|
378
|
+
},
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
event: "api key rotated",
|
|
382
|
+
weight: 1,
|
|
383
|
+
properties: {
|
|
384
|
+
rotation_reason: ["scheduled", "compromised", "policy", "manual"],
|
|
385
|
+
},
|
|
386
|
+
},
|
|
387
|
+
{
|
|
388
|
+
event: "api call",
|
|
389
|
+
weight: 10,
|
|
390
|
+
properties: {
|
|
391
|
+
model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"],
|
|
392
|
+
input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
|
|
393
|
+
output_tokens: u.weighNumRange(10, 4000, 0.4, 500),
|
|
394
|
+
tokens_used: u.weighNumRange(100, 12000, 0.4, 2500),
|
|
395
|
+
cost_usd: [0.001, 0.002, 0.003, 0.003, 0.005, 0.005, 0.005, 0.008, 0.008, 0.01, 0.01, 0.01, 0.01, 0.015, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
|
|
396
|
+
cost_per_token: [0.000002, 0.000003, 0.000005, 0.000005, 0.000008, 0.000008, 0.00001, 0.00001, 0.00001, 0.000012, 0.000015, 0.00002, 0.000025, 0.00003],
|
|
397
|
+
latency_ms: u.weighNumRange(100, 15000, 0.4, 1500),
|
|
398
|
+
cache_enabled: [false],
|
|
399
|
+
is_error: [false],
|
|
400
|
+
error_type: ["none"],
|
|
401
|
+
multi_turn: [false, false, false, true],
|
|
402
|
+
context_window: [200000],
|
|
403
|
+
stream: [true, true, true, false],
|
|
404
|
+
stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
|
|
405
|
+
},
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
event: "tool use call",
|
|
409
|
+
weight: 4,
|
|
410
|
+
properties: {
|
|
411
|
+
tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
|
|
412
|
+
execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
|
|
413
|
+
success: [true, true, true, true, false],
|
|
414
|
+
tool_input_tokens: u.weighNumRange(50, 2000, 0.4, 300),
|
|
415
|
+
tool_output_tokens: u.weighNumRange(20, 5000, 0.4, 500),
|
|
416
|
+
},
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
event: "batch job submitted",
|
|
420
|
+
weight: 2,
|
|
421
|
+
properties: {
|
|
422
|
+
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
423
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
424
|
+
estimated_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
425
|
+
priority: ["standard", "standard", "standard", "express"],
|
|
426
|
+
},
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
event: "batch job completed",
|
|
430
|
+
weight: 2,
|
|
431
|
+
properties: {
|
|
432
|
+
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
433
|
+
processing_time_sec: u.weighNumRange(60, 7200, 0.4, 900),
|
|
434
|
+
total_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
435
|
+
success_rate: u.weighNumRange(90, 100, 0.8, 98),
|
|
436
|
+
},
|
|
437
|
+
},
|
|
438
|
+
{
|
|
439
|
+
event: "eval job",
|
|
440
|
+
weight: 3,
|
|
441
|
+
properties: {
|
|
442
|
+
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
443
|
+
num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
|
|
444
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
445
|
+
dataset_name: ["prod_prompts", "safety_suite", "regression_set", "benchmark_v2", "custom_eval"],
|
|
446
|
+
},
|
|
447
|
+
},
|
|
448
|
+
{
|
|
449
|
+
event: "eval result",
|
|
450
|
+
weight: 3,
|
|
451
|
+
properties: {
|
|
452
|
+
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
453
|
+
score: u.weighNumRange(0, 100, 0.6, 75),
|
|
454
|
+
pass_rate: u.weighNumRange(50, 100, 0.7, 85),
|
|
455
|
+
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
456
|
+
regression_detected: [false, false, false, false, true],
|
|
457
|
+
},
|
|
458
|
+
},
|
|
459
|
+
{
|
|
460
|
+
event: "rate limit error",
|
|
461
|
+
weight: 3,
|
|
462
|
+
properties: {
|
|
463
|
+
error_code: [429],
|
|
464
|
+
retry_after_ms: u.weighNumRange(1000, 60000, 0.3, 5000),
|
|
465
|
+
requests_per_minute: u.weighNumRange(50, 2000, 0.4, 500),
|
|
466
|
+
tier_limit: ["Free", "Build", "Enterprise"],
|
|
467
|
+
},
|
|
468
|
+
},
|
|
469
|
+
{
|
|
470
|
+
event: "billing payment",
|
|
471
|
+
weight: 2,
|
|
472
|
+
properties: {
|
|
473
|
+
amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
|
|
474
|
+
payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
|
|
475
|
+
billing_period: ["monthly", "monthly", "annual"],
|
|
476
|
+
tokens_consumed: u.weighNumRange(100000, 50000000, 0.3, 5000000),
|
|
477
|
+
},
|
|
478
|
+
},
|
|
479
|
+
{
|
|
480
|
+
event: "model selected",
|
|
481
|
+
weight: 3,
|
|
482
|
+
properties: {
|
|
483
|
+
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
484
|
+
is_default: [true, true, false],
|
|
485
|
+
selection_context: ["playground", "api_config", "eval_setup", "batch_config"],
|
|
486
|
+
},
|
|
487
|
+
},
|
|
488
|
+
{
|
|
489
|
+
event: "dashboard viewed",
|
|
490
|
+
weight: 5,
|
|
491
|
+
properties: {
|
|
492
|
+
dashboard_section: ["usage", "billing", "api_keys", "models", "evals", "logs"],
|
|
493
|
+
time_range: ["1h", "24h", "7d", "30d"],
|
|
494
|
+
},
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
event: "docs searched",
|
|
498
|
+
weight: 4,
|
|
499
|
+
properties: {
|
|
500
|
+
search_query_category: ["api_reference", "quickstart", "pricing", "models", "tool_use", "batch_api", "caching", "errors"],
|
|
501
|
+
results_found: u.weighNumRange(0, 50, 0.5, 8),
|
|
502
|
+
clicked_result: [true, true, true, false],
|
|
503
|
+
},
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
event: "member invited",
|
|
507
|
+
weight: 2,
|
|
508
|
+
properties: {
|
|
509
|
+
invite_role: ["admin", "developer", "developer", "billing", "viewer"],
|
|
510
|
+
invite_method: ["email", "email", "sso", "link"],
|
|
511
|
+
},
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
event: "webhook configured",
|
|
515
|
+
weight: 1,
|
|
516
|
+
properties: {
|
|
517
|
+
webhook_event: ["usage_alert", "rate_limit", "batch_complete", "eval_complete", "billing_threshold"],
|
|
518
|
+
delivery_method: ["https", "https", "slack", "email"],
|
|
519
|
+
},
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
event: "playground session",
|
|
523
|
+
weight: 4,
|
|
524
|
+
properties: {
|
|
525
|
+
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
526
|
+
turns: u.weighNumRange(1, 30, 0.4, 5),
|
|
527
|
+
shared: [false, false, false, true],
|
|
528
|
+
tokens_used: u.weighNumRange(100, 20000, 0.3, 3000),
|
|
529
|
+
},
|
|
530
|
+
},
|
|
531
|
+
{
|
|
532
|
+
event: "account deactivated",
|
|
533
|
+
weight: 1,
|
|
534
|
+
isChurnEvent: true,
|
|
535
|
+
returnLikelihood: 0.1,
|
|
536
|
+
isStrictEvent: true,
|
|
537
|
+
properties: {
|
|
538
|
+
reason: ["cost", "switched_provider", "project_ended", "rate_limits", "no_longer_needed", "performance"],
|
|
539
|
+
},
|
|
540
|
+
},
|
|
541
|
+
],
|
|
542
|
+
|
|
543
|
+
// -- Funnels (3) ------------------------------------------
|
|
544
|
+
funnels: [
|
|
545
|
+
{
|
|
546
|
+
name: "Onboarding",
|
|
547
|
+
sequence: ["organization created", "api key created", "api call"],
|
|
548
|
+
conversionRate: 70,
|
|
549
|
+
order: "sequential",
|
|
550
|
+
isFirstFunnel: true,
|
|
551
|
+
timeToConvert: 48,
|
|
552
|
+
weight: 3,
|
|
553
|
+
},
|
|
554
|
+
{
|
|
555
|
+
name: "API to Eval Pipeline",
|
|
556
|
+
sequence: ["api call", "tool use call", "eval job"],
|
|
557
|
+
conversionRate: 45,
|
|
558
|
+
order: "sequential",
|
|
559
|
+
timeToConvert: 168,
|
|
560
|
+
weight: 5,
|
|
561
|
+
},
|
|
562
|
+
{
|
|
563
|
+
name: "Usage to Billing",
|
|
564
|
+
sequence: ["api call", "billing payment"],
|
|
565
|
+
conversionRate: 30,
|
|
566
|
+
order: "sequential",
|
|
567
|
+
timeToConvert: 336,
|
|
568
|
+
weight: 2,
|
|
569
|
+
},
|
|
570
|
+
],
|
|
571
|
+
|
|
572
|
+
// -- SuperProps --------------------------------------------
|
|
573
|
+
superProps: {
|
|
574
|
+
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
575
|
+
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
576
|
+
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
577
|
+
},
|
|
578
|
+
|
|
579
|
+
// -- UserProps ---------------------------------------------
|
|
580
|
+
userProps: {
|
|
581
|
+
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
582
|
+
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
583
|
+
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
584
|
+
monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
|
|
585
|
+
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
586
|
+
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
587
|
+
},
|
|
588
|
+
|
|
589
|
+
// -- Hook Function ----------------------------------------
|
|
590
|
+
hook: function (record, type, meta) {
|
|
591
|
+
// ─────────────────────────────────────────────────────────
|
|
592
|
+
// Hook #6: OUTAGE DAY (event)
|
|
593
|
+
// Days 40-41: 40% of api calls get is_error=true with
|
|
594
|
+
// service error types
|
|
595
|
+
// ─────────────────────────────────────────────────────────
|
|
596
|
+
if (type === "event") {
|
|
597
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
598
|
+
if (record.event === "api call") {
|
|
599
|
+
const eventTime = dayjs(record.time);
|
|
600
|
+
const dayInDataset = eventTime.diff(datasetStart, "days", true);
|
|
601
|
+
|
|
602
|
+
// Hook #6: Outage day errors
|
|
603
|
+
if (dayInDataset >= 40 && dayInDataset < 42) {
|
|
604
|
+
if (chance.bool({ likelihood: 40 })) {
|
|
605
|
+
record.is_error = true;
|
|
606
|
+
record.error_type = chance.pickone([
|
|
607
|
+
"service_overloaded",
|
|
608
|
+
"internal_server_error",
|
|
609
|
+
"gateway_timeout",
|
|
610
|
+
]);
|
|
611
|
+
record.latency_ms = Math.floor((record.latency_ms || 1500) * 3);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// Hook #2 (model migration wave) MOVED to everything hook —
|
|
616
|
+
// at event-hook time `record.api_tier` is the random per-event
|
|
617
|
+
// value, not the user's profile tier. Stamping happens later in
|
|
618
|
+
// the everything hook.
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
return record;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// ─────────────────────────────────────────────────────────
|
|
625
|
+
// Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
626
|
+
// Enterprise users complete API to Eval Pipeline funnel 1.5x faster
|
|
627
|
+
// (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
|
|
628
|
+
// 1.4). Mutates record[i].time. No flag.
|
|
629
|
+
// ─────────────────────────────────────────────────────────
|
|
630
|
+
if (type === "funnel-post") {
|
|
631
|
+
const segment = meta?.profile?.api_tier;
|
|
632
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
633
|
+
const factor = (
|
|
634
|
+
segment === "Enterprise" ? 0.67 :
|
|
635
|
+
segment === "Free" ? 1.4 :
|
|
636
|
+
1.0
|
|
637
|
+
);
|
|
638
|
+
if (factor !== 1.0) {
|
|
639
|
+
for (let i = 1; i < record.length; i++) {
|
|
640
|
+
const prev = dayjs(record[i - 1].time);
|
|
641
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
642
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// ─────────────────────────────────────────────────────────
|
|
649
|
+
// EVERYTHING HOOKS
|
|
650
|
+
// ─────────────────────────────────────────────────────────
|
|
651
|
+
if (type === "everything") {
|
|
652
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
653
|
+
let events = record;
|
|
654
|
+
if (!events.length) return record;
|
|
655
|
+
const profile = meta && meta.profile ? meta.profile : {};
|
|
656
|
+
|
|
657
|
+
// Stamp superProps from profile for consistency
|
|
658
|
+
events.forEach(e => {
|
|
659
|
+
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
660
|
+
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
661
|
+
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
// Determine first event time for relative day calculations
|
|
665
|
+
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
666
|
+
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
667
|
+
|
|
668
|
+
// ─────────────────────────────────────────────────────
|
|
669
|
+
// Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
|
|
670
|
+
// Scale context_window and input_tokens by tier
|
|
671
|
+
// ─────────────────────────────────────────────────────
|
|
672
|
+
const tier = profile.api_tier || "Free";
|
|
673
|
+
const contextWindow = tier === "Enterprise" ? 2000000 : tier === "Build" ? 1000000 : 200000;
|
|
674
|
+
const inputMultiplier = tier === "Enterprise" ? 4 : tier === "Build" ? 2 : 1;
|
|
675
|
+
|
|
676
|
+
events.forEach(e => {
|
|
677
|
+
if (e.event === "api call") {
|
|
678
|
+
e.context_window = contextWindow;
|
|
679
|
+
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
680
|
+
}
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
// ─────────────────────────────────────────────────────
|
|
684
|
+
// Hook #1: PROMPT CACHING ADOPTION (CONVERSION)
|
|
685
|
+
// Users with any cache_enabled=true get 70% cost reduction
|
|
686
|
+
// on all subsequent api calls
|
|
687
|
+
// ─────────────────────────────────────────────────────
|
|
688
|
+
// ~25% of users have caching enabled on at least one event
|
|
689
|
+
const userId = events[0] && events[0].user_id;
|
|
690
|
+
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
691
|
+
const isCacheUser = (idHash % 4) === 0;
|
|
692
|
+
|
|
693
|
+
if (isCacheUser) {
|
|
694
|
+
let cacheActivated = false;
|
|
695
|
+
// Activate caching on events after the first 20% of user events
|
|
696
|
+
const activationPoint = Math.floor(events.length * 0.2);
|
|
697
|
+
events.forEach((e, idx) => {
|
|
698
|
+
if (e.event === "api call") {
|
|
699
|
+
if (idx >= activationPoint) {
|
|
700
|
+
cacheActivated = true;
|
|
701
|
+
}
|
|
702
|
+
if (cacheActivated) {
|
|
703
|
+
e.cache_enabled = true;
|
|
704
|
+
e.cost_usd = Math.round((e.cost_usd || 0.01) * 0.3 * 10000) / 10000;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// ─────────────────────────────────────────────────────
|
|
711
|
+
// Hook #2: MODEL MIGRATION WAVE
|
|
712
|
+
// After day 60, 35% of Build/Enterprise api_calls switch to
|
|
713
|
+
// opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
|
|
714
|
+
// (authoritative) and uses post-shift event timestamps.
|
|
715
|
+
// ─────────────────────────────────────────────────────
|
|
716
|
+
const datasetStartDay60 = datasetStart.add(60, "days");
|
|
717
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
718
|
+
events.forEach(e => {
|
|
719
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
|
|
720
|
+
if (chance.bool({ likelihood: 35 })) {
|
|
721
|
+
e.model = "opus-4-7";
|
|
722
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
});
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// ─────────────────────────────────────────────────────
|
|
729
|
+
// Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
|
|
730
|
+
// Users with tool use + multi_turn get 8x tokens, 3x events
|
|
731
|
+
// ─────────────────────────────────────────────────────
|
|
732
|
+
const hasToolUse = events.some(e => e.event === "tool use call");
|
|
733
|
+
const hasMultiTurn = events.some(e => e.event === "api call" && e.multi_turn === true);
|
|
734
|
+
const isAgenticUser = hasToolUse && hasMultiTurn;
|
|
735
|
+
|
|
736
|
+
if (isAgenticUser) {
|
|
737
|
+
events.forEach(e => {
|
|
738
|
+
if (e.event === "api call") {
|
|
739
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
|
|
740
|
+
}
|
|
741
|
+
});
|
|
742
|
+
|
|
743
|
+
const apiCalls = events.filter(e => e.event === "api call");
|
|
744
|
+
const extraCount = apiCalls.length * 2;
|
|
745
|
+
for (let i = 0; i < extraCount; i++) {
|
|
746
|
+
const template = apiCalls[i % apiCalls.length];
|
|
747
|
+
if (template) {
|
|
748
|
+
events.push({
|
|
749
|
+
...template,
|
|
750
|
+
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
751
|
+
user_id: template.user_id,
|
|
752
|
+
multi_turn: true,
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// ─────────────────────────────────────────────────────
|
|
759
|
+
// Hook #4: RATE LIMIT CHURN
|
|
760
|
+
// >=2 rate limit errors in first 7 days -> remove 60% of
|
|
761
|
+
// events after week 1. Threshold is intentionally low because
|
|
762
|
+
// avgEventsPerUserPerDay=0.83 means most users only generate
|
|
763
|
+
// a handful of events per week.
|
|
764
|
+
// ─────────────────────────────────────────────────────
|
|
765
|
+
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
766
|
+
const earlyRateLimits = events.filter(e =>
|
|
767
|
+
e.event === "rate limit error" &&
|
|
768
|
+
dayjs(e.time).isBefore(firstWeekEnd)
|
|
769
|
+
).length;
|
|
770
|
+
|
|
771
|
+
if (earlyRateLimits >= 2) {
|
|
772
|
+
// Remove 60% of events after week 1
|
|
773
|
+
events = events.filter(e => {
|
|
774
|
+
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
775
|
+
return chance.bool({ likelihood: 40 });
|
|
776
|
+
}
|
|
777
|
+
return true;
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// ─────────────────────────────────────────────────────
|
|
782
|
+
// Hook #7: BATCH API DISCOUNT (PURCHASE VALUE)
|
|
783
|
+
// Batch users get 50% lower cost_per_token, 2x tokens_used
|
|
784
|
+
// ─────────────────────────────────────────────────────
|
|
785
|
+
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
786
|
+
|
|
787
|
+
if (isBatchUser) {
|
|
788
|
+
events.forEach(e => {
|
|
789
|
+
if (e.event === "api call") {
|
|
790
|
+
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
|
|
791
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
|
|
792
|
+
}
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
// ─────────────────────────────────────────────────────
|
|
797
|
+
// Hook #8: EVAL-DRIVEN RETENTION
|
|
798
|
+
// Early eval users (first 7 days) get 75% D30 retention
|
|
799
|
+
// Non-eval users get only 25% D30 retention (remove events)
|
|
800
|
+
// ─────────────────────────────────────────────────────
|
|
801
|
+
const hasEarlyEval = events.some(e =>
|
|
802
|
+
e.event === "eval job" &&
|
|
803
|
+
dayjs(e.time).isBefore(firstWeekEnd)
|
|
804
|
+
);
|
|
805
|
+
|
|
806
|
+
if (hasEarlyEval) {
|
|
807
|
+
// Early eval users keep all their events (high retention)
|
|
808
|
+
} else {
|
|
809
|
+
// Non-eval users: remove 75% of events after day 30
|
|
810
|
+
const day30 = firstEventTime.add(30, "days");
|
|
811
|
+
events = events.filter(e => {
|
|
812
|
+
if (dayjs(e.time).isAfter(day30)) {
|
|
813
|
+
return chance.bool({ likelihood: 25 });
|
|
814
|
+
}
|
|
815
|
+
return true;
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
// ─────────────────────────────────────────────────────
|
|
820
|
+
// Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
|
|
821
|
+
// Count "docs searched" events between first "organization
|
|
822
|
+
// created" (sign-up) and any "billing payment". Sweet 5-8 → +35%
|
|
823
|
+
// on amount_usd of billing-payment events. Over 9+ → drop 30%
|
|
824
|
+
// of billing-payment events.
|
|
825
|
+
// ─────────────────────────────────────────────────────
|
|
826
|
+
const orgEvent = events.find(e => e.event === "organization created");
|
|
827
|
+
const firstBilling = events.find(e => e.event === "billing payment");
|
|
828
|
+
if (orgEvent && firstBilling) {
|
|
829
|
+
const aTime = dayjs(orgEvent.time);
|
|
830
|
+
const bTime = dayjs(firstBilling.time);
|
|
831
|
+
const docsBetween = events.filter(e =>
|
|
832
|
+
e.event === "docs searched" &&
|
|
833
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
834
|
+
dayjs(e.time).isBefore(bTime)
|
|
835
|
+
).length;
|
|
836
|
+
if (docsBetween >= 5 && docsBetween <= 8) {
|
|
837
|
+
events.forEach(e => {
|
|
838
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
839
|
+
e.amount_usd = Math.round(e.amount_usd * 1.35);
|
|
840
|
+
}
|
|
841
|
+
});
|
|
842
|
+
} else if (docsBetween >= 9) {
|
|
843
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
844
|
+
if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
|
|
845
|
+
events.splice(i, 1);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
return events;
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
return record;
|
|
855
|
+
},
|
|
856
|
+
};
|
|
857
|
+
|
|
858
|
+
export default config;
|