@ak--47/dungeon-master 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dungeons/technical/array-of-object-lookup.js +0 -2
- package/dungeons/technical/simple.js +3 -4
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +192 -133
- package/dungeons/vertical/community.js +64 -34
- package/dungeons/vertical/crypto.js +333 -224
- package/dungeons/vertical/dating.js +228 -282
- package/dungeons/vertical/devtools.js +164 -62
- package/dungeons/vertical/ecommerce.js +240 -91
- package/dungeons/vertical/education.js +328 -310
- package/dungeons/vertical/fintech.js +437 -311
- package/dungeons/vertical/fitness.js +141 -58
- package/dungeons/vertical/food-delivery.js +298 -318
- package/dungeons/vertical/gaming.js +378 -231
- package/dungeons/vertical/healthcare.js +140 -60
- package/dungeons/vertical/insurance-application.js +168 -73
- package/dungeons/vertical/logistics.js +95 -6
- package/dungeons/vertical/marketplace.js +137 -51
- package/dungeons/vertical/media.js +241 -386
- package/dungeons/vertical/real-estate.js +322 -315
- package/dungeons/vertical/sass.js +253 -263
- package/dungeons/vertical/social.js +262 -206
- package/dungeons/vertical/travel.js +59 -16
- package/index.js +17 -17
- package/lib/core/config-validator.js +93 -21
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +11 -14
- package/lib/generators/funnels.js +12 -4
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/orchestrators/user-loop.js +47 -47
- package/lib/utils/utils.js +71 -39
- package/package.json +1 -1
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +38 -15
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/vertical/ai-platform-schema.json +0 -617
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/crypto-schema.json +0 -546
- package/dungeons/vertical/dating-schema.json +0 -401
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -2703
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/real-estate-schema.json +0 -527
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -147,8 +147,6 @@ const config = {
|
|
|
147
147
|
}],
|
|
148
148
|
hook: function (record, type, meta) {
|
|
149
149
|
|
|
150
|
-
const NOW = dayjs();
|
|
151
|
-
|
|
152
150
|
if (type === "event") {
|
|
153
151
|
// Pattern 1: Checkouts with coupons get a discount_applied flag and adjusted total
|
|
154
152
|
if (record.event === "checkout" && record.coupon && record.coupon !== "none") {
|
|
@@ -218,10 +218,9 @@ const config = {
|
|
|
218
218
|
lookupTables: [],
|
|
219
219
|
hook: function (record, type, meta) {
|
|
220
220
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
const OVER_THINGS_GET_BETTER = NOW.subtract(15, 'day');
|
|
221
|
+
// Anchor relative dates to the resolved dataset window — never wall-clock.
|
|
222
|
+
const datasetEnd = dayjs.unix(meta.datasetEnd);
|
|
223
|
+
const OVER_THINGS_GET_BETTER = datasetEnd.subtract(15, 'day');
|
|
225
224
|
|
|
226
225
|
if (type === "event") {
|
|
227
226
|
const EVENT_TIME = dayjs(record.time);
|
|
@@ -16,9 +16,6 @@ import * as v from "ak-tools";
|
|
|
16
16
|
|
|
17
17
|
dayjs.extend(utc);
|
|
18
18
|
const chance = u.initChance(SEED);
|
|
19
|
-
const NOW = dayjs();
|
|
20
|
-
const DATASET_START = NOW.subtract(num_days, "days");
|
|
21
|
-
|
|
22
19
|
/** @typedef {import("../../types").Dungeon} Config */
|
|
23
20
|
|
|
24
21
|
/**
|
|
@@ -47,11 +44,15 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
47
44
|
* - multi_turn: whether the call is part of a conversation
|
|
48
45
|
*
|
|
49
46
|
* ===============================================================
|
|
50
|
-
* ANALYTICS HOOKS (
|
|
47
|
+
* ANALYTICS HOOKS (10 hooks)
|
|
51
48
|
* ===============================================================
|
|
52
49
|
*
|
|
50
|
+
* NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
51
|
+
* only via behavioral cohorts (count event per user) or raw-prop
|
|
52
|
+
* breakdowns (api_tier from profile, model, error_type).
|
|
53
|
+
*
|
|
53
54
|
* ---------------------------------------------------------------
|
|
54
|
-
* 1. PROMPT CACHING ADOPTION (CONVERSION —
|
|
55
|
+
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
55
56
|
* ---------------------------------------------------------------
|
|
56
57
|
*
|
|
57
58
|
* PATTERN: Customers who enable prompt caching see 70% lower
|
|
@@ -108,60 +109,44 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
108
109
|
* waves among power users who want improved capabilities.
|
|
109
110
|
*
|
|
110
111
|
* ---------------------------------------------------------------
|
|
111
|
-
* 3. AGENTIC LOOP POWER USERS (
|
|
112
|
+
* 3. AGENTIC LOOP POWER USERS (everything)
|
|
112
113
|
* ---------------------------------------------------------------
|
|
113
114
|
*
|
|
114
|
-
* PATTERN: Users
|
|
115
|
-
*
|
|
116
|
-
*
|
|
115
|
+
* PATTERN: Users with both "tool use call" AND any api-call event with
|
|
116
|
+
* multi_turn=true get 8x tokens_used on api calls plus 2 extra cloned
|
|
117
|
+
* api-call events per existing (3x rate). Cloned events with unique
|
|
118
|
+
* offset timestamps. No flag — discover via cohort builder.
|
|
117
119
|
*
|
|
118
120
|
* HOW TO FIND IT IN MIXPANEL:
|
|
119
121
|
*
|
|
120
|
-
* Report 1:
|
|
121
|
-
* - Report type: Insights
|
|
122
|
+
* Report 1: Tokens per User — Agentic Cohort
|
|
123
|
+
* - Report type: Insights (with cohort)
|
|
124
|
+
* - Cohort A: users with both >= 1 "tool use call" AND >= 1 api-call with multi_turn=true
|
|
125
|
+
* - Cohort B: rest
|
|
122
126
|
* - Event: "api call"
|
|
123
127
|
* - Measure: Average of "tokens_used"
|
|
124
|
-
* -
|
|
125
|
-
* - Expected: is_agentic_user=true ~ 8x tokens (agentic ~ 40K, standard ~ 5K)
|
|
128
|
+
* - Expected: A ~ 8x B
|
|
126
129
|
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
* - Event: "api call"
|
|
130
|
-
* - Measure: Total per user (average)
|
|
131
|
-
* - Breakdown: "is_agentic_user"
|
|
132
|
-
* - Expected: agentic users ~ 3x more api calls
|
|
133
|
-
*
|
|
134
|
-
* REAL-WORLD ANALOGUE: Agentic workloads (coding agents, research
|
|
135
|
-
* assistants) consume dramatically more tokens via extended tool-use
|
|
136
|
-
* loops and multi-turn conversations.
|
|
130
|
+
* REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
|
|
131
|
+
* tokens via extended tool-use loops.
|
|
137
132
|
*
|
|
138
133
|
* ---------------------------------------------------------------
|
|
139
|
-
* 4. RATE LIMIT CHURN (
|
|
134
|
+
* 4. RATE LIMIT CHURN (everything)
|
|
140
135
|
* ---------------------------------------------------------------
|
|
141
136
|
*
|
|
142
|
-
* PATTERN: Users
|
|
143
|
-
*
|
|
144
|
-
* churn from frustration.
|
|
137
|
+
* PATTERN: Users with >= 2 "rate limit error" events in first 7 days
|
|
138
|
+
* lose 60% of events after week 1. No flag — discover via cohort.
|
|
145
139
|
*
|
|
146
140
|
* HOW TO FIND IT IN MIXPANEL:
|
|
147
141
|
*
|
|
148
|
-
* Report 1: Retention by Early Rate
|
|
142
|
+
* Report 1: Retention by Early Rate-Limit Cohort
|
|
149
143
|
* - Report type: Retention
|
|
150
|
-
* -
|
|
151
|
-
* -
|
|
152
|
-
* -
|
|
153
|
-
* - Expected: hit_rate_limit_early=true ~ 40% D30 retention
|
|
154
|
-
* vs ~80% for others
|
|
155
|
-
*
|
|
156
|
-
* Report 2: Event Volume Post Rate-Limit
|
|
157
|
-
* - Report type: Insights
|
|
158
|
-
* - Event: any event
|
|
159
|
-
* - Measure: Total per user (average)
|
|
160
|
-
* - Breakdown: "hit_rate_limit_early"
|
|
161
|
-
* - Expected: rate-limited users ~ 40% of normal volume
|
|
144
|
+
* - Cohort A: users with >= 2 "rate limit error" in first 7 days
|
|
145
|
+
* - Cohort B: rest
|
|
146
|
+
* - Expected: A retention ~ 40% vs B ~ 80%
|
|
162
147
|
*
|
|
163
|
-
* REAL-WORLD ANALOGUE: Developers who get rate-limited early
|
|
164
|
-
*
|
|
148
|
+
* REAL-WORLD ANALOGUE: Developers who get rate-limited early often
|
|
149
|
+
* switch to a competitor.
|
|
165
150
|
*
|
|
166
151
|
* ---------------------------------------------------------------
|
|
167
152
|
* 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
|
|
@@ -220,59 +205,96 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
220
205
|
* that spike error rates across all customers.
|
|
221
206
|
*
|
|
222
207
|
* ---------------------------------------------------------------
|
|
223
|
-
* 7. BATCH API DISCOUNT (
|
|
208
|
+
* 7. BATCH API DISCOUNT (everything)
|
|
224
209
|
* ---------------------------------------------------------------
|
|
225
210
|
*
|
|
226
|
-
* PATTERN: Users
|
|
227
|
-
* on api calls
|
|
228
|
-
*
|
|
211
|
+
* PATTERN: Users with any "batch job submitted" event get 50% lower
|
|
212
|
+
* cost_per_token on api calls + 2x tokens_used. Mutates raw props.
|
|
213
|
+
* No flag — discover via cohort builder.
|
|
229
214
|
*
|
|
230
215
|
* HOW TO FIND IT IN MIXPANEL:
|
|
231
216
|
*
|
|
232
|
-
* Report 1: Cost
|
|
233
|
-
* - Report type: Insights
|
|
217
|
+
* Report 1: Cost per Token by Batch Cohort
|
|
218
|
+
* - Report type: Insights (with cohort)
|
|
219
|
+
* - Cohort A: users with >= 1 "batch job submitted"
|
|
220
|
+
* - Cohort B: rest
|
|
234
221
|
* - Event: "api call"
|
|
235
222
|
* - Measure: Average of "cost_per_token"
|
|
236
|
-
* -
|
|
237
|
-
* - Expected: is_batch_user=true ~ 50% lower cost per token
|
|
223
|
+
* - Expected: A ~ 0.5x B
|
|
238
224
|
*
|
|
239
|
-
*
|
|
240
|
-
* - Report type: Insights
|
|
241
|
-
* - Event: "api call"
|
|
242
|
-
* - Measure: Average of "tokens_used"
|
|
243
|
-
* - Breakdown: "is_batch_user"
|
|
244
|
-
* - Expected: batch users ~ 2x token volume
|
|
245
|
-
*
|
|
246
|
-
* REAL-WORLD ANALOGUE: Batch API pricing incentivizes high-volume
|
|
247
|
-
* workloads with discounted per-token rates.
|
|
225
|
+
* REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
|
|
248
226
|
*
|
|
249
227
|
* ---------------------------------------------------------------
|
|
250
|
-
* 8. EVAL-DRIVEN RETENTION (
|
|
228
|
+
* 8. EVAL-DRIVEN RETENTION (everything)
|
|
251
229
|
* ---------------------------------------------------------------
|
|
252
230
|
*
|
|
253
|
-
* PATTERN: Users
|
|
254
|
-
*
|
|
255
|
-
*
|
|
231
|
+
* PATTERN: Users with any "eval job" in first 7 days keep all events.
|
|
232
|
+
* Non-eval users lose 75% of post-day-30 events. No flag — discover
|
|
233
|
+
* via retention cohort.
|
|
256
234
|
*
|
|
257
235
|
* HOW TO FIND IT IN MIXPANEL:
|
|
258
236
|
*
|
|
259
|
-
* Report 1: Retention by Early Eval
|
|
237
|
+
* Report 1: Retention by Early Eval Cohort
|
|
260
238
|
* - Report type: Retention
|
|
261
|
-
* -
|
|
262
|
-
* -
|
|
263
|
-
* -
|
|
264
|
-
* - Expected: has_early_eval=true ~ 75% D30 vs 25% for false
|
|
239
|
+
* - Cohort A: users with >= 1 "eval job" in first 7 days
|
|
240
|
+
* - Cohort B: rest
|
|
241
|
+
* - Expected: A ~ 75% D30 vs B ~ 25%
|
|
265
242
|
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
243
|
+
* REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
|
|
244
|
+
*
|
|
245
|
+
* ---------------------------------------------------------------
|
|
246
|
+
* 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
247
|
+
* ---------------------------------------------------------------
|
|
248
|
+
*
|
|
249
|
+
* PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
|
|
250
|
+
* 1.5x faster than baseline (factor 0.67 on inter-event gaps); Free
|
|
251
|
+
* users 1.4x slower (factor 1.4). Mutates funnel event timestamps.
|
|
252
|
+
*
|
|
253
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
254
|
+
*
|
|
255
|
+
* Report 1: API to Eval — Median Time-to-Convert by Tier
|
|
256
|
+
* - Report type: Funnels
|
|
257
|
+
* - Steps: "api call" -> "tool use call" -> "eval job"
|
|
258
|
+
* - Measure: Median time to convert
|
|
259
|
+
* - Breakdown: "api_tier"
|
|
260
|
+
* - Expected: Enterprise ~ 0.67x Build; Free ~ 1.4x Build
|
|
261
|
+
*
|
|
262
|
+
* NOTE (funnel-post measurement): visible only via Mixpanel funnel
|
|
263
|
+
* median TTC. Cross-event MIN→MIN SQL queries on raw events do NOT
|
|
264
|
+
* show this — funnel-post adjusts gaps within funnel instances, not
|
|
265
|
+
* across the user's full event history.
|
|
266
|
+
*
|
|
267
|
+
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
|
|
268
|
+
* who execute end-to-end pipelines faster.
|
|
269
|
+
*
|
|
270
|
+
* ---------------------------------------------------------------
|
|
271
|
+
* 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
|
|
272
|
+
* ---------------------------------------------------------------
|
|
273
|
+
*
|
|
274
|
+
* PATTERN: Count "docs searched" events between organization-created and
|
|
275
|
+
* first billing-payment. Sweet 5-8 → +35% on amount_usd of billing
|
|
276
|
+
* payment events. Over 9+ → drop 30% of billing payment events. No flag.
|
|
277
|
+
*
|
|
278
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
273
279
|
*
|
|
274
|
-
*
|
|
275
|
-
*
|
|
280
|
+
* Report 1: Avg Billing Amount by Docs-Searched Bucket
|
|
281
|
+
* - Report type: Insights (with cohort)
|
|
282
|
+
* - Cohort A: users with 5-8 "docs searched" between sign-up and first billing
|
|
283
|
+
* - Cohort B: users with 0-4
|
|
284
|
+
* - Event: "billing payment"
|
|
285
|
+
* - Measure: Average of "amount_usd"
|
|
286
|
+
* - Expected: A ~ 1.35x B
|
|
287
|
+
*
|
|
288
|
+
* Report 2: Billing Payments per User on Heavy Searchers
|
|
289
|
+
* - Report type: Insights (with cohort)
|
|
290
|
+
* - Cohort C: users with >= 9 "docs searched" between sign-up and billing
|
|
291
|
+
* - Cohort A: users with 5-8
|
|
292
|
+
* - Event: "billing payment"
|
|
293
|
+
* - Measure: Total per user
|
|
294
|
+
* - Expected: C ~ 30% fewer billing payments per user
|
|
295
|
+
*
|
|
296
|
+
* REAL-WORLD ANALOGUE: Reading the docs lifts willingness to pay; doc
|
|
297
|
+
* obsession signals stuck on integration and never paying.
|
|
276
298
|
*
|
|
277
299
|
* ===============================================================
|
|
278
300
|
* EXPECTED METRICS SUMMARY
|
|
@@ -281,20 +303,25 @@ const DATASET_START = NOW.subtract(num_days, "days");
|
|
|
281
303
|
* Hook | Metric | Baseline | Effect | Ratio
|
|
282
304
|
* ----------------------------|----------------------|------------|--------------|------
|
|
283
305
|
* Prompt Caching Adoption | cost_usd | $0.01 | $0.003 | 0.3x
|
|
284
|
-
* Model Migration Wave | opus-4-7 share
|
|
285
|
-
* Agentic Loop Power Users |
|
|
306
|
+
* Model Migration Wave | opus-4-7 share paid | 0% | ~ 35% | new
|
|
307
|
+
* Agentic Loop Power Users | tokens/api-call | 1x | 8x | 8x
|
|
286
308
|
* Rate Limit Churn | D30 retention | 80% | 40% | 0.5x
|
|
287
309
|
* Tier-Based Context Window | input_tokens | 2K (Free) | 8K (Ent) | 4x
|
|
288
|
-
* Outage Day | error rate
|
|
289
|
-
* Batch API Discount | cost_per_token |
|
|
310
|
+
* Outage Day | error rate days 40-41| 5% | 40% | 8x
|
|
311
|
+
* Batch API Discount | cost_per_token | 1x | 0.5x | -50%
|
|
290
312
|
* Eval-Driven Retention | D30 retention | 25% | 75% | 3x
|
|
313
|
+
* API-to-Eval T2C | median min by tier | 1x (Build) | 0.67x / 1.4x | 1.5x range
|
|
314
|
+
* Docs Magic Number | sweet billing amount | 1x | 1.35x | 1.35x
|
|
315
|
+
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
291
316
|
*/
|
|
292
317
|
|
|
293
318
|
/** @type {Config} */
|
|
294
319
|
const config = {
|
|
295
320
|
token,
|
|
296
321
|
seed: SEED,
|
|
297
|
-
|
|
322
|
+
datasetStart: "2026-01-01T00:00:00Z",
|
|
323
|
+
datasetEnd: "2026-04-28T23:59:59Z",
|
|
324
|
+
// numDays: num_days,
|
|
298
325
|
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
299
326
|
numUsers: num_users,
|
|
300
327
|
hasAnonIds: false,
|
|
@@ -373,8 +400,6 @@ const config = {
|
|
|
373
400
|
error_type: ["none"],
|
|
374
401
|
multi_turn: [false, false, false, true],
|
|
375
402
|
context_window: [200000],
|
|
376
|
-
is_agentic_user: [false],
|
|
377
|
-
is_batch_user: [false],
|
|
378
403
|
stream: [true, true, true, false],
|
|
379
404
|
stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
|
|
380
405
|
},
|
|
@@ -559,9 +584,6 @@ const config = {
|
|
|
559
584
|
monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
|
|
560
585
|
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
561
586
|
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
562
|
-
has_eval_pipeline: [false],
|
|
563
|
-
hit_rate_limit_early: [false],
|
|
564
|
-
has_early_eval: [false],
|
|
565
587
|
},
|
|
566
588
|
|
|
567
589
|
// -- Hook Function ----------------------------------------
|
|
@@ -572,9 +594,10 @@ const config = {
|
|
|
572
594
|
// service error types
|
|
573
595
|
// ─────────────────────────────────────────────────────────
|
|
574
596
|
if (type === "event") {
|
|
597
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
575
598
|
if (record.event === "api call") {
|
|
576
599
|
const eventTime = dayjs(record.time);
|
|
577
|
-
const dayInDataset = eventTime.diff(
|
|
600
|
+
const dayInDataset = eventTime.diff(datasetStart, "days", true);
|
|
578
601
|
|
|
579
602
|
// Hook #6: Outage day errors
|
|
580
603
|
if (dayInDataset >= 40 && dayInDataset < 42) {
|
|
@@ -589,36 +612,44 @@ const config = {
|
|
|
589
612
|
}
|
|
590
613
|
}
|
|
591
614
|
|
|
592
|
-
// Hook #2
|
|
593
|
-
//
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
(record.api_tier === "Build" || record.api_tier === "Enterprise") &&
|
|
597
|
-
chance.bool({ likelihood: 35 })
|
|
598
|
-
) {
|
|
599
|
-
record.model = "opus-4-7";
|
|
600
|
-
}
|
|
601
|
-
}
|
|
615
|
+
// Hook #2 (model migration wave) MOVED to everything hook —
|
|
616
|
+
// at event-hook time `record.api_tier` is the random per-event
|
|
617
|
+
// value, not the user's profile tier. Stamping happens later in
|
|
618
|
+
// the everything hook.
|
|
602
619
|
}
|
|
603
620
|
|
|
604
621
|
return record;
|
|
605
622
|
}
|
|
606
623
|
|
|
607
624
|
// ─────────────────────────────────────────────────────────
|
|
608
|
-
// Hook:
|
|
609
|
-
//
|
|
625
|
+
// Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
626
|
+
// Enterprise users complete API to Eval Pipeline funnel 1.5x faster
|
|
627
|
+
// (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
|
|
628
|
+
// 1.4). Mutates record[i].time. No flag.
|
|
610
629
|
// ─────────────────────────────────────────────────────────
|
|
611
|
-
if (type === "
|
|
612
|
-
|
|
613
|
-
record.
|
|
614
|
-
|
|
615
|
-
|
|
630
|
+
if (type === "funnel-post") {
|
|
631
|
+
const segment = meta?.profile?.api_tier;
|
|
632
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
633
|
+
const factor = (
|
|
634
|
+
segment === "Enterprise" ? 0.67 :
|
|
635
|
+
segment === "Free" ? 1.4 :
|
|
636
|
+
1.0
|
|
637
|
+
);
|
|
638
|
+
if (factor !== 1.0) {
|
|
639
|
+
for (let i = 1; i < record.length; i++) {
|
|
640
|
+
const prev = dayjs(record[i - 1].time);
|
|
641
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
642
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
616
646
|
}
|
|
617
647
|
|
|
618
648
|
// ─────────────────────────────────────────────────────────
|
|
619
649
|
// EVERYTHING HOOKS
|
|
620
650
|
// ─────────────────────────────────────────────────────────
|
|
621
651
|
if (type === "everything") {
|
|
652
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
622
653
|
let events = record;
|
|
623
654
|
if (!events.length) return record;
|
|
624
655
|
const profile = meta && meta.profile ? meta.profile : {};
|
|
@@ -632,7 +663,7 @@ const config = {
|
|
|
632
663
|
|
|
633
664
|
// Determine first event time for relative day calculations
|
|
634
665
|
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
635
|
-
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) :
|
|
666
|
+
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
636
667
|
|
|
637
668
|
// ─────────────────────────────────────────────────────
|
|
638
669
|
// Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
|
|
@@ -677,15 +708,22 @@ const config = {
|
|
|
677
708
|
}
|
|
678
709
|
|
|
679
710
|
// ─────────────────────────────────────────────────────
|
|
680
|
-
// Hook #2: MODEL MIGRATION WAVE
|
|
681
|
-
//
|
|
682
|
-
//
|
|
711
|
+
// Hook #2: MODEL MIGRATION WAVE
|
|
712
|
+
// After day 60, 35% of Build/Enterprise api_calls switch to
|
|
713
|
+
// opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
|
|
714
|
+
// (authoritative) and uses post-shift event timestamps.
|
|
683
715
|
// ─────────────────────────────────────────────────────
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
716
|
+
const datasetStartDay60 = datasetStart.add(60, "days");
|
|
717
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
718
|
+
events.forEach(e => {
|
|
719
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
|
|
720
|
+
if (chance.bool({ likelihood: 35 })) {
|
|
721
|
+
e.model = "opus-4-7";
|
|
722
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
});
|
|
726
|
+
}
|
|
689
727
|
|
|
690
728
|
// ─────────────────────────────────────────────────────
|
|
691
729
|
// Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
|
|
@@ -696,17 +734,14 @@ const config = {
|
|
|
696
734
|
const isAgenticUser = hasToolUse && hasMultiTurn;
|
|
697
735
|
|
|
698
736
|
if (isAgenticUser) {
|
|
699
|
-
// Mark all api calls as agentic and boost tokens
|
|
700
737
|
events.forEach(e => {
|
|
701
738
|
if (e.event === "api call") {
|
|
702
|
-
e.is_agentic_user = true;
|
|
703
739
|
e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
|
|
704
740
|
}
|
|
705
741
|
});
|
|
706
742
|
|
|
707
|
-
// Inject 3x extra api call events by cloning existing ones
|
|
708
743
|
const apiCalls = events.filter(e => e.event === "api call");
|
|
709
|
-
const extraCount = apiCalls.length * 2;
|
|
744
|
+
const extraCount = apiCalls.length * 2;
|
|
710
745
|
for (let i = 0; i < extraCount; i++) {
|
|
711
746
|
const template = apiCalls[i % apiCalls.length];
|
|
712
747
|
if (template) {
|
|
@@ -714,7 +749,6 @@ const config = {
|
|
|
714
749
|
...template,
|
|
715
750
|
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
716
751
|
user_id: template.user_id,
|
|
717
|
-
is_agentic_user: true,
|
|
718
752
|
multi_turn: true,
|
|
719
753
|
});
|
|
720
754
|
}
|
|
@@ -723,8 +757,10 @@ const config = {
|
|
|
723
757
|
|
|
724
758
|
// ─────────────────────────────────────────────────────
|
|
725
759
|
// Hook #4: RATE LIMIT CHURN
|
|
726
|
-
// >=
|
|
727
|
-
// events after week 1
|
|
760
|
+
// >=2 rate limit errors in first 7 days -> remove 60% of
|
|
761
|
+
// events after week 1. Threshold is intentionally low because
|
|
762
|
+
// avgEventsPerUserPerDay=0.83 means most users only generate
|
|
763
|
+
// a handful of events per week.
|
|
728
764
|
// ─────────────────────────────────────────────────────
|
|
729
765
|
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
730
766
|
const earlyRateLimits = events.filter(e =>
|
|
@@ -732,14 +768,11 @@ const config = {
|
|
|
732
768
|
dayjs(e.time).isBefore(firstWeekEnd)
|
|
733
769
|
).length;
|
|
734
770
|
|
|
735
|
-
if (earlyRateLimits >=
|
|
736
|
-
// Tag the user profile
|
|
737
|
-
if (profile) profile.hit_rate_limit_early = true;
|
|
738
|
-
|
|
771
|
+
if (earlyRateLimits >= 2) {
|
|
739
772
|
// Remove 60% of events after week 1
|
|
740
773
|
events = events.filter(e => {
|
|
741
774
|
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
742
|
-
return chance.bool({ likelihood: 40 });
|
|
775
|
+
return chance.bool({ likelihood: 40 });
|
|
743
776
|
}
|
|
744
777
|
return true;
|
|
745
778
|
});
|
|
@@ -754,7 +787,6 @@ const config = {
|
|
|
754
787
|
if (isBatchUser) {
|
|
755
788
|
events.forEach(e => {
|
|
756
789
|
if (e.event === "api call") {
|
|
757
|
-
e.is_batch_user = true;
|
|
758
790
|
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
|
|
759
791
|
e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
|
|
760
792
|
}
|
|
@@ -772,23 +804,50 @@ const config = {
|
|
|
772
804
|
);
|
|
773
805
|
|
|
774
806
|
if (hasEarlyEval) {
|
|
775
|
-
// Tag user profile
|
|
776
|
-
if (profile) {
|
|
777
|
-
profile.has_early_eval = true;
|
|
778
|
-
profile.has_eval_pipeline = true;
|
|
779
|
-
}
|
|
780
807
|
// Early eval users keep all their events (high retention)
|
|
781
808
|
} else {
|
|
782
809
|
// Non-eval users: remove 75% of events after day 30
|
|
783
810
|
const day30 = firstEventTime.add(30, "days");
|
|
784
811
|
events = events.filter(e => {
|
|
785
812
|
if (dayjs(e.time).isAfter(day30)) {
|
|
786
|
-
return chance.bool({ likelihood: 25 });
|
|
813
|
+
return chance.bool({ likelihood: 25 });
|
|
787
814
|
}
|
|
788
815
|
return true;
|
|
789
816
|
});
|
|
790
817
|
}
|
|
791
818
|
|
|
819
|
+
// ─────────────────────────────────────────────────────
|
|
820
|
+
// Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
|
|
821
|
+
// Count "docs searched" events between first "organization
|
|
822
|
+
// created" (sign-up) and any "billing payment". Sweet 5-8 → +35%
|
|
823
|
+
// on amount_usd of billing-payment events. Over 9+ → drop 30%
|
|
824
|
+
// of billing-payment events.
|
|
825
|
+
// ─────────────────────────────────────────────────────
|
|
826
|
+
const orgEvent = events.find(e => e.event === "organization created");
|
|
827
|
+
const firstBilling = events.find(e => e.event === "billing payment");
|
|
828
|
+
if (orgEvent && firstBilling) {
|
|
829
|
+
const aTime = dayjs(orgEvent.time);
|
|
830
|
+
const bTime = dayjs(firstBilling.time);
|
|
831
|
+
const docsBetween = events.filter(e =>
|
|
832
|
+
e.event === "docs searched" &&
|
|
833
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
834
|
+
dayjs(e.time).isBefore(bTime)
|
|
835
|
+
).length;
|
|
836
|
+
if (docsBetween >= 5 && docsBetween <= 8) {
|
|
837
|
+
events.forEach(e => {
|
|
838
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
839
|
+
e.amount_usd = Math.round(e.amount_usd * 1.35);
|
|
840
|
+
}
|
|
841
|
+
});
|
|
842
|
+
} else if (docsBetween >= 9) {
|
|
843
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
844
|
+
if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
|
|
845
|
+
events.splice(i, 1);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
792
851
|
return events;
|
|
793
852
|
}
|
|
794
853
|
|