@ak--47/dungeon-master 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dungeons/technical/array-of-object-lookup.js +0 -2
- package/dungeons/technical/hook-helpers-verify.js +89 -0
- package/dungeons/technical/identity-model-verify.js +47 -0
- package/dungeons/technical/pattern-aggregate-by-bin.js +41 -0
- package/dungeons/technical/pattern-attributed-by-source.js +42 -0
- package/dungeons/technical/pattern-frequency-by-frequency.js +40 -0
- package/dungeons/technical/pattern-funnel-frequency.js +54 -0
- package/dungeons/technical/pattern-ttc-by-segment.js +45 -0
- package/dungeons/technical/simple.js +3 -4
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +216 -164
- package/dungeons/vertical/community.js +74 -41
- package/dungeons/vertical/crypto.js +343 -233
- package/dungeons/vertical/dating.js +246 -292
- package/dungeons/vertical/devtools.js +179 -70
- package/dungeons/vertical/ecommerce.js +275 -122
- package/dungeons/vertical/education.js +345 -312
- package/dungeons/vertical/fintech.js +445 -314
- package/dungeons/vertical/fitness.js +199 -165
- package/dungeons/vertical/food-delivery.js +309 -330
- package/dungeons/vertical/gaming.js +387 -241
- package/dungeons/vertical/healthcare.js +149 -66
- package/dungeons/vertical/insurance-application.js +173 -75
- package/dungeons/vertical/logistics.js +107 -12
- package/dungeons/vertical/marketplace.js +171 -76
- package/dungeons/vertical/media.js +259 -402
- package/dungeons/vertical/real-estate.js +336 -318
- package/dungeons/vertical/sass.js +294 -288
- package/dungeons/vertical/social.js +289 -234
- package/dungeons/vertical/travel.js +66 -20
- package/index.js +17 -17
- package/lib/core/config-validator.js +226 -175
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +57 -104
- package/lib/generators/funnels.js +212 -93
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/hook-helpers/_internal.js +23 -0
- package/lib/hook-helpers/cohort.js +124 -0
- package/lib/hook-helpers/identity.js +56 -0
- package/lib/hook-helpers/index.js +44 -0
- package/lib/hook-helpers/inject.js +99 -0
- package/lib/hook-helpers/mutate.js +151 -0
- package/lib/hook-helpers/timing.js +99 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +38 -0
- package/lib/hook-patterns/attributed-by-source.js +72 -0
- package/lib/hook-patterns/frequency-by-frequency.js +46 -0
- package/lib/hook-patterns/funnel-frequency-breakdown.js +73 -0
- package/lib/hook-patterns/index.js +14 -0
- package/lib/hook-patterns/time-to-convert-by-segment.js +41 -0
- package/lib/orchestrators/user-loop.js +143 -293
- package/lib/utils/utils.js +97 -52
- package/lib/verify/emulate-breakdown.js +281 -0
- package/lib/verify/index.js +12 -0
- package/lib/verify/verify-dungeon.js +61 -0
- package/package.json +6 -4
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +434 -225
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/vertical/ai-platform-schema.json +0 -617
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/crypto-schema.json +0 -546
- package/dungeons/vertical/dating-schema.json +0 -401
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -2703
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/real-estate-schema.json +0 -527
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// ── TWEAK THESE ──
|
|
2
2
|
const SEED = "promptforge";
|
|
3
3
|
const num_days = 120;
|
|
4
|
-
const num_users =
|
|
4
|
+
const num_users = 10_000;
|
|
5
5
|
const avg_events_per_user_per_day = 0.83;
|
|
6
6
|
let token = "your-mixpanel-token";
|
|
7
7
|
|
|
@@ -16,9 +16,6 @@ import * as v from "ak-tools";
|
|
|
16
16
|
|
|
17
17
|
dayjs.extend(utc);
|
|
18
18
|
const chance = u.initChance(SEED);
|
|
19
|
-
const NOW = dayjs();
|
|
20
|
-
const DATASET_START = NOW.subtract(num_days, "days");
|
|
21
|
-
|
|
22
19
|
/** @typedef {import("../../types").Dungeon} Config */
|
|
23
20
|
|
|
24
21
|
/**
|
|
@@ -47,11 +44,15 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
47
44
|
* - multi_turn: whether the call is part of a conversation
|
|
48
45
|
*
|
|
49
46
|
* ===============================================================
|
|
50
|
-
* ANALYTICS HOOKS (
|
|
47
|
+
* ANALYTICS HOOKS (10 hooks)
|
|
51
48
|
* ===============================================================
|
|
52
49
|
*
|
|
50
|
+
* NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
51
|
+
* only via behavioral cohorts (count event per user) or raw-prop
|
|
52
|
+
* breakdowns (api_tier from profile, model, error_type).
|
|
53
|
+
*
|
|
53
54
|
* ---------------------------------------------------------------
|
|
54
|
-
* 1. PROMPT CACHING ADOPTION (CONVERSION —
|
|
55
|
+
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
55
56
|
* ---------------------------------------------------------------
|
|
56
57
|
*
|
|
57
58
|
* PATTERN: Customers who enable prompt caching see 70% lower
|
|
@@ -108,60 +109,44 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
108
109
|
* waves among power users who want improved capabilities.
|
|
109
110
|
*
|
|
110
111
|
* ---------------------------------------------------------------
|
|
111
|
-
* 3. AGENTIC LOOP POWER USERS (
|
|
112
|
+
* 3. AGENTIC LOOP POWER USERS (everything)
|
|
112
113
|
* ---------------------------------------------------------------
|
|
113
114
|
*
|
|
114
|
-
* PATTERN: Users
|
|
115
|
-
*
|
|
116
|
-
*
|
|
115
|
+
* PATTERN: Users with 3+ "tool use call" AND 3+ api-call events with
|
|
116
|
+
* multi_turn=true get 8x tokens_used on api calls plus 2 extra cloned
|
|
117
|
+
* api-call events per existing (3x rate). Cloned events with unique
|
|
118
|
+
* offset timestamps. No flag — discover via cohort builder.
|
|
117
119
|
*
|
|
118
120
|
* HOW TO FIND IT IN MIXPANEL:
|
|
119
121
|
*
|
|
120
|
-
* Report 1:
|
|
121
|
-
* - Report type: Insights
|
|
122
|
+
* Report 1: Tokens per User — Agentic Cohort
|
|
123
|
+
* - Report type: Insights (with cohort)
|
|
124
|
+
* - Cohort A: users with >= 3 "tool use call" AND >= 3 api-call with multi_turn=true
|
|
125
|
+
* - Cohort B: rest
|
|
122
126
|
* - Event: "api call"
|
|
123
127
|
* - Measure: Average of "tokens_used"
|
|
124
|
-
* -
|
|
125
|
-
* - Expected: is_agentic_user=true ~ 8x tokens (agentic ~ 40K, standard ~ 5K)
|
|
126
|
-
*
|
|
127
|
-
* Report 2: API Call Volume — Agentic vs Standard
|
|
128
|
-
* - Report type: Insights
|
|
129
|
-
* - Event: "api call"
|
|
130
|
-
* - Measure: Total per user (average)
|
|
131
|
-
* - Breakdown: "is_agentic_user"
|
|
132
|
-
* - Expected: agentic users ~ 3x more api calls
|
|
128
|
+
* - Expected: A ~ 8x B
|
|
133
129
|
*
|
|
134
|
-
* REAL-WORLD ANALOGUE: Agentic workloads
|
|
135
|
-
*
|
|
136
|
-
* loops and multi-turn conversations.
|
|
130
|
+
* REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
|
|
131
|
+
* tokens via extended tool-use loops.
|
|
137
132
|
*
|
|
138
133
|
* ---------------------------------------------------------------
|
|
139
|
-
* 4. RATE LIMIT CHURN (
|
|
134
|
+
* 4. RATE LIMIT CHURN (everything)
|
|
140
135
|
* ---------------------------------------------------------------
|
|
141
136
|
*
|
|
142
|
-
* PATTERN: Users
|
|
143
|
-
*
|
|
144
|
-
* churn from frustration.
|
|
137
|
+
* PATTERN: Users with >= 2 "rate limit error" events in first 7 days
|
|
138
|
+
* lose 60% of events after week 1. No flag — discover via cohort.
|
|
145
139
|
*
|
|
146
140
|
* HOW TO FIND IT IN MIXPANEL:
|
|
147
141
|
*
|
|
148
|
-
* Report 1: Retention by Early Rate
|
|
142
|
+
* Report 1: Retention by Early Rate-Limit Cohort
|
|
149
143
|
* - Report type: Retention
|
|
150
|
-
* -
|
|
151
|
-
* -
|
|
152
|
-
* -
|
|
153
|
-
* - Expected: hit_rate_limit_early=true ~ 40% D30 retention
|
|
154
|
-
* vs ~80% for others
|
|
155
|
-
*
|
|
156
|
-
* Report 2: Event Volume Post Rate-Limit
|
|
157
|
-
* - Report type: Insights
|
|
158
|
-
* - Event: any event
|
|
159
|
-
* - Measure: Total per user (average)
|
|
160
|
-
* - Breakdown: "hit_rate_limit_early"
|
|
161
|
-
* - Expected: rate-limited users ~ 40% of normal volume
|
|
144
|
+
* - Cohort A: users with >= 2 "rate limit error" in first 7 days
|
|
145
|
+
* - Cohort B: rest
|
|
146
|
+
* - Expected: A retention ~ 40% vs B ~ 80%
|
|
162
147
|
*
|
|
163
|
-
* REAL-WORLD ANALOGUE: Developers who get rate-limited early
|
|
164
|
-
*
|
|
148
|
+
* REAL-WORLD ANALOGUE: Developers who get rate-limited early often
|
|
149
|
+
* switch to a competitor.
|
|
165
150
|
*
|
|
166
151
|
* ---------------------------------------------------------------
|
|
167
152
|
* 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
|
|
@@ -220,59 +205,96 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
220
205
|
* that spike error rates across all customers.
|
|
221
206
|
*
|
|
222
207
|
* ---------------------------------------------------------------
|
|
223
|
-
* 7. BATCH API DISCOUNT (
|
|
208
|
+
* 7. BATCH API DISCOUNT (everything)
|
|
224
209
|
* ---------------------------------------------------------------
|
|
225
210
|
*
|
|
226
|
-
* PATTERN: Users
|
|
227
|
-
* on api calls
|
|
228
|
-
*
|
|
211
|
+
* PATTERN: Users with any "batch job submitted" event get 50% lower
|
|
212
|
+
* cost_per_token on api calls + 2x tokens_used. Mutates raw props.
|
|
213
|
+
* No flag — discover via cohort builder.
|
|
229
214
|
*
|
|
230
215
|
* HOW TO FIND IT IN MIXPANEL:
|
|
231
216
|
*
|
|
232
|
-
* Report 1: Cost
|
|
233
|
-
* - Report type: Insights
|
|
217
|
+
* Report 1: Cost per Token by Batch Cohort
|
|
218
|
+
* - Report type: Insights (with cohort)
|
|
219
|
+
* - Cohort A: users with >= 1 "batch job submitted"
|
|
220
|
+
* - Cohort B: rest
|
|
234
221
|
* - Event: "api call"
|
|
235
222
|
* - Measure: Average of "cost_per_token"
|
|
236
|
-
* -
|
|
237
|
-
* - Expected: is_batch_user=true ~ 50% lower cost per token
|
|
238
|
-
*
|
|
239
|
-
* Report 2: Token Volume — Batch Users
|
|
240
|
-
* - Report type: Insights
|
|
241
|
-
* - Event: "api call"
|
|
242
|
-
* - Measure: Average of "tokens_used"
|
|
243
|
-
* - Breakdown: "is_batch_user"
|
|
244
|
-
* - Expected: batch users ~ 2x token volume
|
|
223
|
+
* - Expected: A ~ 0.5x B
|
|
245
224
|
*
|
|
246
|
-
* REAL-WORLD ANALOGUE: Batch API pricing
|
|
247
|
-
* workloads with discounted per-token rates.
|
|
225
|
+
* REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
|
|
248
226
|
*
|
|
249
227
|
* ---------------------------------------------------------------
|
|
250
|
-
* 8. EVAL-DRIVEN RETENTION (
|
|
228
|
+
* 8. EVAL-DRIVEN RETENTION (everything)
|
|
251
229
|
* ---------------------------------------------------------------
|
|
252
230
|
*
|
|
253
|
-
* PATTERN: Users
|
|
254
|
-
*
|
|
255
|
-
*
|
|
231
|
+
* PATTERN: Users with any "eval job" in first 7 days keep all events.
|
|
232
|
+
* Non-eval users lose 75% of post-day-30 events. No flag — discover
|
|
233
|
+
* via retention cohort.
|
|
256
234
|
*
|
|
257
235
|
* HOW TO FIND IT IN MIXPANEL:
|
|
258
236
|
*
|
|
259
|
-
* Report 1: Retention by Early Eval
|
|
237
|
+
* Report 1: Retention by Early Eval Cohort
|
|
260
238
|
* - Report type: Retention
|
|
261
|
-
* -
|
|
262
|
-
* -
|
|
263
|
-
* -
|
|
264
|
-
* - Expected: has_early_eval=true ~ 75% D30 vs 25% for false
|
|
239
|
+
* - Cohort A: users with >= 1 "eval job" in first 7 days
|
|
240
|
+
* - Cohort B: rest
|
|
241
|
+
* - Expected: A ~ 75% D30 vs B ~ 25%
|
|
265
242
|
*
|
|
266
|
-
*
|
|
267
|
-
* - Report type: Insights
|
|
268
|
-
* - Event: any event
|
|
269
|
-
* - Measure: Total per user (average)
|
|
270
|
-
* - Breakdown: "has_early_eval"
|
|
271
|
-
* - Line chart by week
|
|
272
|
-
* - Expected: early eval users sustain volume; non-eval users decay
|
|
243
|
+
* REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
|
|
273
244
|
*
|
|
274
|
-
*
|
|
275
|
-
*
|
|
245
|
+
* ---------------------------------------------------------------
|
|
246
|
+
* 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
247
|
+
* ---------------------------------------------------------------
|
|
248
|
+
*
|
|
249
|
+
* PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
|
|
250
|
+
* 1.5x faster than baseline (factor 0.67 on inter-event gaps); Free
|
|
251
|
+
* users 1.4x slower (factor 1.4). Mutates funnel event timestamps.
|
|
252
|
+
*
|
|
253
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
254
|
+
*
|
|
255
|
+
* Report 1: API to Eval — Median Time-to-Convert by Tier
|
|
256
|
+
* - Report type: Funnels
|
|
257
|
+
* - Steps: "api call" -> "tool use call" -> "eval job"
|
|
258
|
+
* - Measure: Median time to convert
|
|
259
|
+
* - Breakdown: "api_tier"
|
|
260
|
+
* - Expected: Enterprise ~ 0.67x Build; Free ~ 1.4x Build
|
|
261
|
+
*
|
|
262
|
+
* NOTE (funnel-post measurement): visible only via Mixpanel funnel
|
|
263
|
+
* median TTC. Cross-event MIN→MIN SQL queries on raw events do NOT
|
|
264
|
+
* show this — funnel-post adjusts gaps within funnel instances, not
|
|
265
|
+
* across the user's full event history.
|
|
266
|
+
*
|
|
267
|
+
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
|
|
268
|
+
* who execute end-to-end pipelines faster.
|
|
269
|
+
*
|
|
270
|
+
* ---------------------------------------------------------------
|
|
271
|
+
* 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
|
|
272
|
+
* ---------------------------------------------------------------
|
|
273
|
+
*
|
|
274
|
+
* PATTERN: Count "docs searched" events between organization-created and
|
|
275
|
+
* first billing-payment. Sweet 2-4 → +35% on amount_usd of billing
|
|
276
|
+
* payment events. Over 5+ → drop 30% of billing payment events. No flag.
|
|
277
|
+
*
|
|
278
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
279
|
+
*
|
|
280
|
+
* Report 1: Avg Billing Amount by Docs-Searched Bucket
|
|
281
|
+
* - Report type: Insights (with cohort)
|
|
282
|
+
* - Cohort A: users with 2-4 "docs searched" between sign-up and first billing
|
|
283
|
+
* - Cohort B: users with 0-1
|
|
284
|
+
* - Event: "billing payment"
|
|
285
|
+
* - Measure: Average of "amount_usd"
|
|
286
|
+
* - Expected: A ~ 1.35x B
|
|
287
|
+
*
|
|
288
|
+
* Report 2: Billing Payments per User on Heavy Searchers
|
|
289
|
+
* - Report type: Insights (with cohort)
|
|
290
|
+
* - Cohort C: users with >= 5 "docs searched" between sign-up and billing
|
|
291
|
+
* - Cohort A: users with 2-4
|
|
292
|
+
* - Event: "billing payment"
|
|
293
|
+
* - Measure: Total per user
|
|
294
|
+
* - Expected: C ~ 30% fewer billing payments per user
|
|
295
|
+
*
|
|
296
|
+
* REAL-WORLD ANALOGUE: Reading the docs lifts willingness to pay; doc
|
|
297
|
+
* obsession signals stuck on integration and never paying.
|
|
276
298
|
*
|
|
277
299
|
* ===============================================================
|
|
278
300
|
* EXPECTED METRICS SUMMARY
|
|
@@ -281,24 +303,31 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
281
303
|
* Hook | Metric | Baseline | Effect | Ratio
|
|
282
304
|
* ----------------------------|----------------------|------------|--------------|------
|
|
283
305
|
* Prompt Caching Adoption | cost_usd | $0.01 | $0.003 | 0.3x
|
|
284
|
-
* Model Migration Wave | opus-4-7 share
|
|
285
|
-
* Agentic Loop Power Users |
|
|
306
|
+
* Model Migration Wave | opus-4-7 share paid | 0% | ~ 35% | new
|
|
307
|
+
* Agentic Loop Power Users | tokens/api-call | 1x | 8x | 8x
|
|
286
308
|
* Rate Limit Churn | D30 retention | 80% | 40% | 0.5x
|
|
287
309
|
* Tier-Based Context Window | input_tokens | 2K (Free) | 8K (Ent) | 4x
|
|
288
|
-
* Outage Day | error rate
|
|
289
|
-
* Batch API Discount | cost_per_token |
|
|
310
|
+
* Outage Day | error rate days 40-41| 5% | 40% | 8x
|
|
311
|
+
* Batch API Discount | cost_per_token | 1x | 0.5x | -50%
|
|
290
312
|
* Eval-Driven Retention | D30 retention | 25% | 75% | 3x
|
|
313
|
+
* API-to-Eval T2C | median min by tier | 1x (Build) | 0.67x / 1.4x | 1.5x range
|
|
314
|
+
* Docs Magic Number | sweet billing amount | 1x | 1.35x | 1.35x
|
|
315
|
+
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
291
316
|
*/
|
|
292
317
|
|
|
293
318
|
/** @type {Config} */
|
|
294
319
|
const config = {
|
|
320
|
+
version: 2,
|
|
295
321
|
token,
|
|
296
322
|
seed: SEED,
|
|
297
|
-
|
|
323
|
+
datasetStart: "2026-01-01T00:00:00Z",
|
|
324
|
+
datasetEnd: "2026-05-01T23:59:59Z",
|
|
325
|
+
// numDays: num_days,
|
|
298
326
|
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
299
327
|
numUsers: num_users,
|
|
300
|
-
hasAnonIds:
|
|
301
|
-
|
|
328
|
+
hasAnonIds: true,
|
|
329
|
+
avgDevicePerUser: 2,
|
|
330
|
+
hasSessionIds: true,
|
|
302
331
|
format: "json",
|
|
303
332
|
gzip: true,
|
|
304
333
|
alsoInferFunnels: false,
|
|
@@ -337,6 +366,7 @@ const config = {
|
|
|
337
366
|
event: "organization created",
|
|
338
367
|
weight: 1,
|
|
339
368
|
isFirstEvent: true,
|
|
369
|
+
isAuthEvent: true,
|
|
340
370
|
properties: {
|
|
341
371
|
org_size: ["solo", "startup", "growth", "enterprise"],
|
|
342
372
|
referral_source: ["docs", "blog", "github", "word_of_mouth", "search", "conference"],
|
|
@@ -373,8 +403,6 @@ const config = {
|
|
|
373
403
|
error_type: ["none"],
|
|
374
404
|
multi_turn: [false, false, false, true],
|
|
375
405
|
context_window: [200000],
|
|
376
|
-
is_agentic_user: [false],
|
|
377
|
-
is_batch_user: [false],
|
|
378
406
|
stream: [true, true, true, false],
|
|
379
407
|
stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
|
|
380
408
|
},
|
|
@@ -559,66 +587,40 @@ const config = {
|
|
|
559
587
|
monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
|
|
560
588
|
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
561
589
|
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
562
|
-
has_eval_pipeline: [false],
|
|
563
|
-
hit_rate_limit_early: [false],
|
|
564
|
-
has_early_eval: [false],
|
|
565
590
|
},
|
|
566
591
|
|
|
567
592
|
// -- Hook Function ----------------------------------------
|
|
568
593
|
hook: function (record, type, meta) {
|
|
594
|
+
|
|
569
595
|
// ─────────────────────────────────────────────────────────
|
|
570
|
-
// Hook
|
|
571
|
-
//
|
|
572
|
-
//
|
|
596
|
+
// Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
597
|
+
// Enterprise users complete API to Eval Pipeline funnel 1.5x faster
|
|
598
|
+
// (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
|
|
599
|
+
// 1.4). Mutates record[i].time. No flag.
|
|
573
600
|
// ─────────────────────────────────────────────────────────
|
|
574
|
-
if (type === "
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
const
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
]);
|
|
588
|
-
record.latency_ms = Math.floor((record.latency_ms || 1500) * 3);
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
// Hook #2: Model migration wave (event portion)
|
|
593
|
-
// After day 60, 35% of Build/Enterprise users switch to opus-4-7
|
|
594
|
-
if (dayInDataset >= 60) {
|
|
595
|
-
if (
|
|
596
|
-
(record.api_tier === "Build" || record.api_tier === "Enterprise") &&
|
|
597
|
-
chance.bool({ likelihood: 35 })
|
|
598
|
-
) {
|
|
599
|
-
record.model = "opus-4-7";
|
|
601
|
+
if (type === "funnel-post") {
|
|
602
|
+
const segment = meta?.profile?.api_tier;
|
|
603
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
604
|
+
const factor = (
|
|
605
|
+
segment === "Enterprise" ? 0.67 :
|
|
606
|
+
segment === "Free" ? 1.4 :
|
|
607
|
+
1.0
|
|
608
|
+
);
|
|
609
|
+
if (factor !== 1.0) {
|
|
610
|
+
for (let i = 1; i < record.length; i++) {
|
|
611
|
+
const prev = dayjs(record[i - 1].time);
|
|
612
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
613
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
600
614
|
}
|
|
601
615
|
}
|
|
602
616
|
}
|
|
603
|
-
|
|
604
|
-
return record;
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
// ─────────────────────────────────────────────────────────
|
|
608
|
-
// Hook: USER PROFILE ENRICHMENT (user)
|
|
609
|
-
// Tag user profiles for discoverability
|
|
610
|
-
// ─────────────────────────────────────────────────────────
|
|
611
|
-
if (type === "user") {
|
|
612
|
-
// Defaults for hook-driven user properties
|
|
613
|
-
record.hit_rate_limit_early = false;
|
|
614
|
-
record.has_early_eval = false;
|
|
615
|
-
record.has_eval_pipeline = false;
|
|
616
617
|
}
|
|
617
618
|
|
|
618
619
|
// ─────────────────────────────────────────────────────────
|
|
619
620
|
// EVERYTHING HOOKS
|
|
620
621
|
// ─────────────────────────────────────────────────────────
|
|
621
622
|
if (type === "everything") {
|
|
623
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
622
624
|
let events = record;
|
|
623
625
|
if (!events.length) return record;
|
|
624
626
|
const profile = meta && meta.profile ? meta.profile : {};
|
|
@@ -630,9 +632,29 @@ const config = {
|
|
|
630
632
|
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
631
633
|
});
|
|
632
634
|
|
|
635
|
+
// ─────────────────────────────────────────────────────
|
|
636
|
+
// Hook #6: OUTAGE DAY (days 40-41)
|
|
637
|
+
// 40% of api calls get is_error=true with service errors
|
|
638
|
+
// ─────────────────────────────────────────────────────
|
|
639
|
+
events.forEach(e => {
|
|
640
|
+
if (e.event !== "api call") return;
|
|
641
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
642
|
+
if (dayInDataset >= 40 && dayInDataset < 42) {
|
|
643
|
+
if (chance.bool({ likelihood: 40 })) {
|
|
644
|
+
e.is_error = true;
|
|
645
|
+
e.error_type = chance.pickone([
|
|
646
|
+
"service_overloaded",
|
|
647
|
+
"internal_server_error",
|
|
648
|
+
"gateway_timeout",
|
|
649
|
+
]);
|
|
650
|
+
e.latency_ms = Math.floor((e.latency_ms || 1500) * 3);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
});
|
|
654
|
+
|
|
633
655
|
// Determine first event time for relative day calculations
|
|
634
656
|
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
635
|
-
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) :
|
|
657
|
+
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
636
658
|
|
|
637
659
|
// ─────────────────────────────────────────────────────
|
|
638
660
|
// Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
|
|
@@ -677,36 +699,42 @@ const config = {
|
|
|
677
699
|
}
|
|
678
700
|
|
|
679
701
|
// ─────────────────────────────────────────────────────
|
|
680
|
-
// Hook #2: MODEL MIGRATION WAVE
|
|
681
|
-
//
|
|
682
|
-
//
|
|
702
|
+
// Hook #2: MODEL MIGRATION WAVE
|
|
703
|
+
// After day 60, 35% of Build/Enterprise api_calls switch to
|
|
704
|
+
// opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
|
|
705
|
+
// (authoritative) and uses post-shift event timestamps.
|
|
683
706
|
// ─────────────────────────────────────────────────────
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
707
|
+
const datasetStartDay60 = datasetStart.add(60, "days");
|
|
708
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
709
|
+
events.forEach(e => {
|
|
710
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
|
|
711
|
+
if (chance.bool({ likelihood: 35 })) {
|
|
712
|
+
e.model = "opus-4-7";
|
|
713
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
});
|
|
717
|
+
}
|
|
689
718
|
|
|
690
719
|
// ─────────────────────────────────────────────────────
|
|
691
720
|
// Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
|
|
692
|
-
// Users with tool use + multi_turn
|
|
721
|
+
// Users with 3+ tool use calls + 3+ multi_turn api calls
|
|
722
|
+
// get 8x tokens, 3x events. Threshold ensures a meaningful
|
|
723
|
+
// agentic cohort (~20-30% of users).
|
|
693
724
|
// ─────────────────────────────────────────────────────
|
|
694
|
-
const
|
|
695
|
-
const
|
|
696
|
-
const isAgenticUser =
|
|
725
|
+
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
726
|
+
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
727
|
+
const isAgenticUser = toolUseCount >= 3 && multiTurnCount >= 3;
|
|
697
728
|
|
|
698
729
|
if (isAgenticUser) {
|
|
699
|
-
// Mark all api calls as agentic and boost tokens
|
|
700
730
|
events.forEach(e => {
|
|
701
731
|
if (e.event === "api call") {
|
|
702
|
-
e.is_agentic_user = true;
|
|
703
732
|
e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
|
|
704
733
|
}
|
|
705
734
|
});
|
|
706
735
|
|
|
707
|
-
// Inject 3x extra api call events by cloning existing ones
|
|
708
736
|
const apiCalls = events.filter(e => e.event === "api call");
|
|
709
|
-
const extraCount = apiCalls.length * 2;
|
|
737
|
+
const extraCount = apiCalls.length * 2;
|
|
710
738
|
for (let i = 0; i < extraCount; i++) {
|
|
711
739
|
const template = apiCalls[i % apiCalls.length];
|
|
712
740
|
if (template) {
|
|
@@ -714,7 +742,6 @@ const config = {
|
|
|
714
742
|
...template,
|
|
715
743
|
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
716
744
|
user_id: template.user_id,
|
|
717
|
-
is_agentic_user: true,
|
|
718
745
|
multi_turn: true,
|
|
719
746
|
});
|
|
720
747
|
}
|
|
@@ -723,8 +750,10 @@ const config = {
|
|
|
723
750
|
|
|
724
751
|
// ─────────────────────────────────────────────────────
|
|
725
752
|
// Hook #4: RATE LIMIT CHURN
|
|
726
|
-
// >=
|
|
727
|
-
// events after week 1
|
|
753
|
+
// >=2 rate limit errors in first 7 days -> remove 60% of
|
|
754
|
+
// events after week 1. Threshold is intentionally low because
|
|
755
|
+
// avgEventsPerUserPerDay=0.83 means most users only generate
|
|
756
|
+
// a handful of events per week.
|
|
728
757
|
// ─────────────────────────────────────────────────────
|
|
729
758
|
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
730
759
|
const earlyRateLimits = events.filter(e =>
|
|
@@ -732,14 +761,11 @@ const config = {
|
|
|
732
761
|
dayjs(e.time).isBefore(firstWeekEnd)
|
|
733
762
|
).length;
|
|
734
763
|
|
|
735
|
-
if (earlyRateLimits >=
|
|
736
|
-
// Tag the user profile
|
|
737
|
-
if (profile) profile.hit_rate_limit_early = true;
|
|
738
|
-
|
|
764
|
+
if (earlyRateLimits >= 2) {
|
|
739
765
|
// Remove 60% of events after week 1
|
|
740
766
|
events = events.filter(e => {
|
|
741
767
|
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
742
|
-
return chance.bool({ likelihood: 40 });
|
|
768
|
+
return chance.bool({ likelihood: 40 });
|
|
743
769
|
}
|
|
744
770
|
return true;
|
|
745
771
|
});
|
|
@@ -754,7 +780,6 @@ const config = {
|
|
|
754
780
|
if (isBatchUser) {
|
|
755
781
|
events.forEach(e => {
|
|
756
782
|
if (e.event === "api call") {
|
|
757
|
-
e.is_batch_user = true;
|
|
758
783
|
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
|
|
759
784
|
e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
|
|
760
785
|
}
|
|
@@ -772,23 +797,50 @@ const config = {
|
|
|
772
797
|
);
|
|
773
798
|
|
|
774
799
|
if (hasEarlyEval) {
|
|
775
|
-
// Tag user profile
|
|
776
|
-
if (profile) {
|
|
777
|
-
profile.has_early_eval = true;
|
|
778
|
-
profile.has_eval_pipeline = true;
|
|
779
|
-
}
|
|
780
800
|
// Early eval users keep all their events (high retention)
|
|
781
801
|
} else {
|
|
782
802
|
// Non-eval users: remove 75% of events after day 30
|
|
783
803
|
const day30 = firstEventTime.add(30, "days");
|
|
784
804
|
events = events.filter(e => {
|
|
785
805
|
if (dayjs(e.time).isAfter(day30)) {
|
|
786
|
-
return chance.bool({ likelihood: 25 });
|
|
806
|
+
return chance.bool({ likelihood: 25 });
|
|
787
807
|
}
|
|
788
808
|
return true;
|
|
789
809
|
});
|
|
790
810
|
}
|
|
791
811
|
|
|
812
|
+
// ─────────────────────────────────────────────────────
|
|
813
|
+
// Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
|
|
814
|
+
// Count "docs searched" events between first "organization
|
|
815
|
+
// created" (sign-up) and any "billing payment". Sweet 2-4 → +35%
|
|
816
|
+
// on amount_usd of billing-payment events. Over 5+ → drop 30%
|
|
817
|
+
// of billing-payment events.
|
|
818
|
+
// ─────────────────────────────────────────────────────
|
|
819
|
+
const orgEvent = events.find(e => e.event === "organization created");
|
|
820
|
+
const firstBilling = events.find(e => e.event === "billing payment");
|
|
821
|
+
if (orgEvent && firstBilling) {
|
|
822
|
+
const aTime = dayjs(orgEvent.time);
|
|
823
|
+
const bTime = dayjs(firstBilling.time);
|
|
824
|
+
const docsBetween = events.filter(e =>
|
|
825
|
+
e.event === "docs searched" &&
|
|
826
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
827
|
+
dayjs(e.time).isBefore(bTime)
|
|
828
|
+
).length;
|
|
829
|
+
if (docsBetween >= 2 && docsBetween <= 4) {
|
|
830
|
+
events.forEach(e => {
|
|
831
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
832
|
+
e.amount_usd = Math.round(e.amount_usd * 1.35);
|
|
833
|
+
}
|
|
834
|
+
});
|
|
835
|
+
} else if (docsBetween >= 5) {
|
|
836
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
837
|
+
if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
|
|
838
|
+
events.splice(i, 1);
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
792
844
|
return events;
|
|
793
845
|
}
|
|
794
846
|
|