@ak--47/dungeon-master 1.4.5 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +158 -0
- package/.claude/skills/create-dungeon/SKILL.md +464 -0
- package/.claude/skills/verify-dungeon/SKILL.md +157 -0
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
- package/.claude/skills/write-hooks/SKILL.md +468 -0
- package/CHANGELOG.md +182 -0
- package/HOOKS.md +1256 -597
- package/README.md +140 -5
- package/dungeons/technical/ad-spend.js +41 -49
- package/dungeons/technical/anonymous-users.js +38 -36
- package/dungeons/technical/array-of-object-lookup.js +136 -153
- package/dungeons/technical/datagen-v15-verify.js +87 -0
- package/dungeons/technical/experiments.js +42 -40
- package/dungeons/technical/foobar.js +114 -118
- package/dungeons/technical/group-analytics.js +42 -40
- package/dungeons/technical/hook-helpers-verify.js +69 -50
- package/dungeons/technical/identity-model-verify.js +22 -12
- package/dungeons/technical/mirror-strategies.js +37 -39
- package/dungeons/technical/nested-objects.js +119 -118
- package/dungeons/technical/pattern-aggregate-by-bin.js +21 -8
- package/dungeons/technical/pattern-attributed-by-source.js +23 -9
- package/dungeons/technical/pattern-frequency-by-frequency.js +21 -8
- package/dungeons/technical/pattern-funnel-frequency.js +30 -15
- package/dungeons/technical/pattern-ttc-by-segment.js +21 -8
- package/dungeons/technical/retention-cadence.js +115 -112
- package/dungeons/technical/sanity.js +86 -80
- package/dungeons/technical/scale-test.js +34 -38
- package/dungeons/technical/scd.js +111 -128
- package/dungeons/technical/simple.js +134 -141
- package/dungeons/technical/simplest.js +111 -65
- package/dungeons/technical/text-generation.js +110 -146
- package/dungeons/vertical/ai-platform.js +300 -333
- package/dungeons/vertical/community.js +290 -255
- package/dungeons/vertical/crypto.js +400 -391
- package/dungeons/vertical/dating.js +421 -375
- package/dungeons/vertical/devtools.js +346 -298
- package/dungeons/vertical/ecommerce.js +322 -394
- package/dungeons/vertical/education.js +380 -325
- package/dungeons/vertical/fintech.js +371 -325
- package/dungeons/vertical/fitness.js +345 -291
- package/dungeons/vertical/food-delivery.js +352 -307
- package/dungeons/vertical/gaming.js +490 -444
- package/dungeons/vertical/healthcare.js +311 -262
- package/dungeons/vertical/insurance-application.js +437 -409
- package/dungeons/vertical/logistics.js +278 -252
- package/dungeons/vertical/marketplace.js +340 -323
- package/dungeons/vertical/media.js +390 -335
- package/dungeons/vertical/real-estate.js +402 -347
- package/dungeons/vertical/sass.js +331 -333
- package/dungeons/vertical/social.js +377 -316
- package/dungeons/vertical/travel.js +302 -295
- package/index.js +64 -7
- package/lib/core/config-validator.js +378 -17
- package/lib/core/dungeon-loader.js +2 -5
- package/lib/generators/events.js +12 -13
- package/lib/generators/funnels.js +76 -2
- package/lib/hook-helpers/index.js +1 -0
- package/lib/hook-helpers/inject.js +95 -0
- package/lib/orchestrators/mixpanel-sender.js +7 -0
- package/lib/orchestrators/user-loop.js +598 -48
- package/lib/templates/defaults.js +59 -59
- package/lib/templates/macro-presets.js +53 -11
- package/lib/utils/dataset-context.js +103 -0
- package/lib/utils/retention-curve.js +140 -0
- package/lib/utils/utils.js +157 -109
- package/lib/verify/counting.js +360 -0
- package/lib/verify/emulate-breakdown.js +531 -108
- package/lib/verify/funnel-engine.js +539 -0
- package/lib/verify/identity.js +78 -0
- package/lib/verify/index.js +20 -0
- package/lib/verify/schema-validator.js +3 -1
- package/lib/verify/verify-dungeon.js +58 -0
- package/package.json +14 -3
- package/scripts/run-dungeon.mjs +12 -1
- package/types.d.ts +353 -4
- package/scripts/smoke-test-all.mjs +0 -162
|
@@ -1,56 +1,43 @@
|
|
|
1
|
-
// ──
|
|
2
|
-
const SEED = "promptforge";
|
|
3
|
-
const num_days = 120;
|
|
4
|
-
const num_users = 10_000;
|
|
5
|
-
const avg_events_per_user_per_day = 0.83;
|
|
6
|
-
let token = "your-mixpanel-token";
|
|
7
|
-
|
|
8
|
-
// ── env overrides ──
|
|
9
|
-
if (process.env.MP_TOKEN) token = process.env.MP_TOKEN;
|
|
10
|
-
|
|
1
|
+
// ── IMPORTS ──
|
|
11
2
|
import dayjs from "dayjs";
|
|
12
3
|
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
13
5
|
import "dotenv/config";
|
|
14
6
|
import * as u from "../../lib/utils/utils.js";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
7
|
+
/** @typedef {import("../../types").Dungeon} Config */
|
|
8
|
+
|
|
9
|
+
// ── OVERVIEW ──
|
|
10
|
+
/*
|
|
11
|
+
* NAME: PromptForge
|
|
12
|
+
* APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
|
|
13
|
+
* requests for chat completions, embeddings, evaluations, and tool
|
|
14
|
+
* use. Billing is per input/output token. Features: prompt caching,
|
|
15
|
+
* tool use, multi-turn conversations, batch API, model selection,
|
|
16
|
+
* eval pipelines.
|
|
17
|
+
* SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
|
|
18
|
+
* CORE LOOP: organization created → api key created → api call → iterate
|
|
19
|
+
*
|
|
20
|
+
* EVENTS (18):
|
|
21
|
+
* api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
|
|
22
|
+
* > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
|
|
23
|
+
* > model selected (3) > api key created (2) > batch job submitted (2)
|
|
24
|
+
* > batch job completed (2) > billing payment (2) > member invited (2)
|
|
25
|
+
* > organization created (1) > api key rotated (1) > webhook configured (1)
|
|
26
|
+
* > account deactivated (1)
|
|
27
|
+
*
|
|
28
|
+
* FUNNELS (3):
|
|
29
|
+
* - Onboarding: organization created → api key created → api call (70%)
|
|
30
|
+
* - API to Eval Pipeline: api call → tool use call → eval job (45%)
|
|
31
|
+
* - Usage to Billing: api call → billing payment (30%)
|
|
32
|
+
*
|
|
33
|
+
* USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
|
|
34
|
+
* SUPER PROPS: api_tier, primary_use_case, sdk_language
|
|
35
|
+
* SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
|
|
36
|
+
* GROUPS: none
|
|
37
|
+
*/
|
|
20
38
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
* DATASET OVERVIEW
|
|
24
|
-
* ===============================================================
|
|
25
|
-
*
|
|
26
|
-
* PromptForge -- an LLM API platform (like Anthropic/OpenAI).
|
|
27
|
-
* Customers (developers and companies) send API requests for chat
|
|
28
|
-
* completions, embeddings, evaluations, and tool use. Billing is
|
|
29
|
-
* per input/output token. Key features: prompt caching, tool use,
|
|
30
|
-
* multi-turn conversations, batch API, model selection, and
|
|
31
|
-
* evaluation pipelines.
|
|
32
|
-
*
|
|
33
|
-
* - 8,000 users over 120 days, ~800K events
|
|
34
|
-
* - Three API tiers: Free, Build, Enterprise
|
|
35
|
-
* - Core loop: org created -> api key created -> api call -> iterate
|
|
36
|
-
* - Revenue: token-based billing with tier-based pricing
|
|
37
|
-
*
|
|
38
|
-
* Key entities:
|
|
39
|
-
* - model: LLM model version (sonnet-4, haiku-4, opus-4-6, opus-4-7)
|
|
40
|
-
* - api_tier: Free / Build / Enterprise (determines context window, rate limits)
|
|
41
|
-
* - tokens_used: total tokens consumed per API call (input + output)
|
|
42
|
-
* - cost_usd: dollar cost of a single API call
|
|
43
|
-
* - cache_enabled: prompt caching flag that reduces cost 70%
|
|
44
|
-
* - multi_turn: whether the call is part of a conversation
|
|
45
|
-
*
|
|
46
|
-
* ===============================================================
|
|
47
|
-
* ANALYTICS HOOKS (10 hooks)
|
|
48
|
-
* ===============================================================
|
|
49
|
-
*
|
|
50
|
-
* NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
51
|
-
* only via behavioral cohorts (count event per user) or raw-prop
|
|
52
|
-
* breakdowns (api_tier from profile, model, error_type).
|
|
53
|
-
*
|
|
39
|
+
// ── HOOK STORIES ──
|
|
40
|
+
/*
|
|
54
41
|
* ---------------------------------------------------------------
|
|
55
42
|
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
56
43
|
* ---------------------------------------------------------------
|
|
@@ -315,36 +302,273 @@ const chance = u.initChance(SEED);
|
|
|
315
302
|
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
316
303
|
*/
|
|
317
304
|
|
|
305
|
+
// ── SCALE ──
|
|
306
|
+
const SEED = "promptforge";
|
|
307
|
+
const NUM_USERS = 10_000;
|
|
308
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
309
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
310
|
+
const EVENTS_PER_DAY = 0.83;
|
|
311
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
312
|
+
|
|
313
|
+
const chance = u.initChance(SEED);
|
|
314
|
+
|
|
315
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
316
|
+
const OUTAGE_START_DAY = 40;
|
|
317
|
+
const OUTAGE_END_DAY = 42;
|
|
318
|
+
const OUTAGE_ERROR_LIKELIHOOD = 40;
|
|
319
|
+
const OUTAGE_LATENCY_MULT = 3;
|
|
320
|
+
|
|
321
|
+
const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
|
|
322
|
+
const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
|
|
323
|
+
|
|
324
|
+
const CACHE_USER_HASH_MOD = 4;
|
|
325
|
+
const CACHE_COST_FACTOR = 0.3;
|
|
326
|
+
const CACHE_ACTIVATION_PCT = 0.2;
|
|
327
|
+
|
|
328
|
+
const MODEL_MIGRATION_DAY = 60;
|
|
329
|
+
const MODEL_MIGRATION_LIKELIHOOD = 35;
|
|
330
|
+
const MODEL_MIGRATION_TOKEN_MULT = 1.5;
|
|
331
|
+
|
|
332
|
+
const AGENTIC_TOOL_THRESHOLD = 3;
|
|
333
|
+
const AGENTIC_MULTITURN_THRESHOLD = 3;
|
|
334
|
+
const AGENTIC_TOKEN_MULT = 8;
|
|
335
|
+
const AGENTIC_CLONE_MULT = 2;
|
|
336
|
+
|
|
337
|
+
const RATE_LIMIT_THRESHOLD = 2;
|
|
338
|
+
const RATE_LIMIT_KEEP_LIKELIHOOD = 40;
|
|
339
|
+
|
|
340
|
+
const BATCH_COST_FACTOR = 0.5;
|
|
341
|
+
const BATCH_TOKEN_MULT = 2;
|
|
342
|
+
|
|
343
|
+
const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
|
|
344
|
+
const EVAL_CUTOFF_DAYS = 30;
|
|
345
|
+
|
|
346
|
+
const DOCS_SWEET_MIN = 2;
|
|
347
|
+
const DOCS_SWEET_MAX = 4;
|
|
348
|
+
const DOCS_OVER_THRESHOLD = 5;
|
|
349
|
+
const DOCS_BILLING_BOOST = 1.35;
|
|
350
|
+
const DOCS_BILLING_DROP_LIKELIHOOD = 30;
|
|
351
|
+
|
|
352
|
+
const FUNNEL_TTC_ENTERPRISE = 0.67;
|
|
353
|
+
const FUNNEL_TTC_FREE = 1.4;
|
|
354
|
+
|
|
355
|
+
// ── HELPER FUNCTIONS ──
|
|
356
|
+
function handleFunnelPostHooks(record, meta) {
|
|
357
|
+
// H9: API-to-Eval TTC scaled by tier
|
|
358
|
+
const segment = meta?.profile?.api_tier;
|
|
359
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
360
|
+
const factor = (
|
|
361
|
+
segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
|
|
362
|
+
segment === "Free" ? FUNNEL_TTC_FREE :
|
|
363
|
+
1.0
|
|
364
|
+
);
|
|
365
|
+
if (factor !== 1.0) {
|
|
366
|
+
for (let i = 1; i < record.length; i++) {
|
|
367
|
+
const prev = dayjs(record[i - 1].time);
|
|
368
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
369
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return record;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function handleEverythingHooks(record, meta) {
|
|
377
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
378
|
+
let events = record;
|
|
379
|
+
if (!events.length) return record;
|
|
380
|
+
const profile = meta && meta.profile ? meta.profile : {};
|
|
381
|
+
|
|
382
|
+
events.forEach(e => {
|
|
383
|
+
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
384
|
+
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
385
|
+
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// H6: Outage day (days 40-41) — 40% of api calls flipped to errors
|
|
389
|
+
events.forEach(e => {
|
|
390
|
+
if (e.event !== "api call") return;
|
|
391
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
392
|
+
if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
|
|
393
|
+
if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
|
|
394
|
+
e.is_error = true;
|
|
395
|
+
e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
|
|
396
|
+
e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
402
|
+
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
403
|
+
|
|
404
|
+
// H5: Tier-based context window & input tokens
|
|
405
|
+
const tier = profile.api_tier || "Free";
|
|
406
|
+
const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
|
|
407
|
+
const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
|
|
408
|
+
events.forEach(e => {
|
|
409
|
+
if (e.event === "api call") {
|
|
410
|
+
e.context_window = contextWindow;
|
|
411
|
+
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
// H1: Prompt caching adoption — ~25% of users; activates 20% into stream
|
|
416
|
+
const userId = events[0] && events[0].user_id;
|
|
417
|
+
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
418
|
+
const isCacheUser = (idHash % CACHE_USER_HASH_MOD) === 0;
|
|
419
|
+
if (isCacheUser) {
|
|
420
|
+
let cacheActivated = false;
|
|
421
|
+
const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
|
|
422
|
+
events.forEach((e, idx) => {
|
|
423
|
+
if (e.event === "api call") {
|
|
424
|
+
if (idx >= activationPoint) cacheActivated = true;
|
|
425
|
+
if (cacheActivated) {
|
|
426
|
+
e.cache_enabled = true;
|
|
427
|
+
e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// H2: Model migration wave — post-day-60 Build/Enterprise calls migrate to opus-4-7
|
|
434
|
+
const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
|
|
435
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
436
|
+
events.forEach(e => {
|
|
437
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
|
|
438
|
+
if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
|
|
439
|
+
e.model = "opus-4-7";
|
|
440
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens, 3x events
|
|
447
|
+
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
448
|
+
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
449
|
+
const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
|
|
450
|
+
if (isAgenticUser) {
|
|
451
|
+
events.forEach(e => {
|
|
452
|
+
if (e.event === "api call") {
|
|
453
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
const apiCalls = events.filter(e => e.event === "api call");
|
|
457
|
+
const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
|
|
458
|
+
for (let i = 0; i < extraCount; i++) {
|
|
459
|
+
const template = apiCalls[i % apiCalls.length];
|
|
460
|
+
if (template) {
|
|
461
|
+
events.push({
|
|
462
|
+
...template,
|
|
463
|
+
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
464
|
+
user_id: template.user_id,
|
|
465
|
+
multi_turn: true,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// H4: Rate-limit churn — 2+ early rate-limit errors → drop 60% of post-week-1 events
|
|
472
|
+
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
473
|
+
const earlyRateLimits = events.filter(e =>
|
|
474
|
+
e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
475
|
+
).length;
|
|
476
|
+
if (earlyRateLimits >= RATE_LIMIT_THRESHOLD) {
|
|
477
|
+
events = events.filter(e => {
|
|
478
|
+
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
479
|
+
return chance.bool({ likelihood: RATE_LIMIT_KEEP_LIKELIHOOD });
|
|
480
|
+
}
|
|
481
|
+
return true;
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// H7: Batch API discount — any batch job submitted → 50% cost, 2x tokens
|
|
486
|
+
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
487
|
+
if (isBatchUser) {
|
|
488
|
+
events.forEach(e => {
|
|
489
|
+
if (e.event === "api call") {
|
|
490
|
+
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
|
|
491
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// H8: Eval-driven retention — non-eval users lose 75% post-day-30
|
|
497
|
+
const hasEarlyEval = events.some(e =>
|
|
498
|
+
e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
499
|
+
);
|
|
500
|
+
if (!hasEarlyEval) {
|
|
501
|
+
const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
|
|
502
|
+
events = events.filter(e => {
|
|
503
|
+
if (dayjs(e.time).isAfter(cutoff)) {
|
|
504
|
+
return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
|
|
505
|
+
}
|
|
506
|
+
return true;
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// H10: Docs-searched magic number — count between org-created and first billing
|
|
511
|
+
const orgEvent = events.find(e => e.event === "organization created");
|
|
512
|
+
const firstBilling = events.find(e => e.event === "billing payment");
|
|
513
|
+
if (orgEvent && firstBilling) {
|
|
514
|
+
const aTime = dayjs(orgEvent.time);
|
|
515
|
+
const bTime = dayjs(firstBilling.time);
|
|
516
|
+
const docsBetween = events.filter(e =>
|
|
517
|
+
e.event === "docs searched" &&
|
|
518
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
519
|
+
dayjs(e.time).isBefore(bTime)
|
|
520
|
+
).length;
|
|
521
|
+
if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
|
|
522
|
+
events.forEach(e => {
|
|
523
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
524
|
+
e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
} else if (docsBetween >= DOCS_OVER_THRESHOLD) {
|
|
528
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
529
|
+
if (events[i].event === "billing payment" && chance.bool({ likelihood: DOCS_BILLING_DROP_LIKELIHOOD })) {
|
|
530
|
+
events.splice(i, 1);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
return events;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// ── CONFIG ──
|
|
318
540
|
/** @type {Config} */
|
|
319
541
|
const config = {
|
|
320
542
|
version: 2,
|
|
321
|
-
token,
|
|
322
543
|
seed: SEED,
|
|
323
|
-
datasetStart:
|
|
324
|
-
datasetEnd:
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
numUsers: num_users,
|
|
328
|
-
hasAnonIds: true,
|
|
329
|
-
avgDevicePerUser: 2,
|
|
330
|
-
hasSessionIds: true,
|
|
544
|
+
datasetStart: DATASET_START,
|
|
545
|
+
datasetEnd: DATASET_END,
|
|
546
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
547
|
+
numUsers: NUM_USERS,
|
|
331
548
|
format: "json",
|
|
332
549
|
gzip: true,
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
550
|
+
credentials: {
|
|
551
|
+
token,
|
|
552
|
+
},
|
|
553
|
+
switches: {
|
|
554
|
+
hasSessionIds: true,
|
|
555
|
+
alsoInferFunnels: false,
|
|
556
|
+
hasLocation: true,
|
|
557
|
+
hasAndroidDevices: false,
|
|
558
|
+
hasIOSDevices: false,
|
|
559
|
+
hasDesktopDevices: true,
|
|
560
|
+
hasBrowser: true,
|
|
561
|
+
hasCampaigns: false,
|
|
562
|
+
isAnonymous: false,
|
|
563
|
+
hasAdSpend: false,
|
|
564
|
+
hasAvatar: true,
|
|
565
|
+
},
|
|
566
|
+
identity: {
|
|
567
|
+
avgDevicePerUser: 2,
|
|
568
|
+
},
|
|
343
569
|
concurrency: 1,
|
|
344
570
|
writeToDisk: false,
|
|
345
|
-
|
|
346
571
|
soup: "growth",
|
|
347
|
-
|
|
348
572
|
scdProps: {
|
|
349
573
|
monthly_api_usage: {
|
|
350
574
|
values: u.weighNumRange(0, 1000000, 0.3, 50),
|
|
@@ -359,8 +583,6 @@ const config = {
|
|
|
359
583
|
max: 6,
|
|
360
584
|
},
|
|
361
585
|
},
|
|
362
|
-
|
|
363
|
-
// -- Events (18) ------------------------------------------
|
|
364
586
|
events: [
|
|
365
587
|
{
|
|
366
588
|
event: "organization created",
|
|
@@ -390,6 +612,7 @@ const config = {
|
|
|
390
612
|
{
|
|
391
613
|
event: "api call",
|
|
392
614
|
weight: 10,
|
|
615
|
+
isStrictEvent: false,
|
|
393
616
|
properties: {
|
|
394
617
|
model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"],
|
|
395
618
|
input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
|
|
@@ -410,6 +633,7 @@ const config = {
|
|
|
410
633
|
{
|
|
411
634
|
event: "tool use call",
|
|
412
635
|
weight: 4,
|
|
636
|
+
isStrictEvent: false,
|
|
413
637
|
properties: {
|
|
414
638
|
tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
|
|
415
639
|
execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
|
|
@@ -441,6 +665,7 @@ const config = {
|
|
|
441
665
|
{
|
|
442
666
|
event: "eval job",
|
|
443
667
|
weight: 3,
|
|
668
|
+
isStrictEvent: false,
|
|
444
669
|
properties: {
|
|
445
670
|
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
446
671
|
num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
|
|
@@ -472,6 +697,7 @@ const config = {
|
|
|
472
697
|
{
|
|
473
698
|
event: "billing payment",
|
|
474
699
|
weight: 2,
|
|
700
|
+
isStrictEvent: false,
|
|
475
701
|
properties: {
|
|
476
702
|
amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
|
|
477
703
|
payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
|
|
@@ -542,8 +768,6 @@ const config = {
|
|
|
542
768
|
},
|
|
543
769
|
},
|
|
544
770
|
],
|
|
545
|
-
|
|
546
|
-
// -- Funnels (3) ------------------------------------------
|
|
547
771
|
funnels: [
|
|
548
772
|
{
|
|
549
773
|
name: "Onboarding",
|
|
@@ -571,15 +795,11 @@ const config = {
|
|
|
571
795
|
weight: 2,
|
|
572
796
|
},
|
|
573
797
|
],
|
|
574
|
-
|
|
575
|
-
// -- SuperProps --------------------------------------------
|
|
576
798
|
superProps: {
|
|
577
799
|
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
578
800
|
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
579
801
|
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
580
802
|
},
|
|
581
|
-
|
|
582
|
-
// -- UserProps ---------------------------------------------
|
|
583
803
|
userProps: {
|
|
584
804
|
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
585
805
|
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
@@ -588,262 +808,9 @@ const config = {
|
|
|
588
808
|
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
589
809
|
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
590
810
|
},
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
// ─────────────────────────────────────────────────────────
|
|
596
|
-
// Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
597
|
-
// Enterprise users complete API to Eval Pipeline funnel 1.5x faster
|
|
598
|
-
// (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
|
|
599
|
-
// 1.4). Mutates record[i].time. No flag.
|
|
600
|
-
// ─────────────────────────────────────────────────────────
|
|
601
|
-
if (type === "funnel-post") {
|
|
602
|
-
const segment = meta?.profile?.api_tier;
|
|
603
|
-
if (Array.isArray(record) && record.length > 1) {
|
|
604
|
-
const factor = (
|
|
605
|
-
segment === "Enterprise" ? 0.67 :
|
|
606
|
-
segment === "Free" ? 1.4 :
|
|
607
|
-
1.0
|
|
608
|
-
);
|
|
609
|
-
if (factor !== 1.0) {
|
|
610
|
-
for (let i = 1; i < record.length; i++) {
|
|
611
|
-
const prev = dayjs(record[i - 1].time);
|
|
612
|
-
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
613
|
-
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
// ─────────────────────────────────────────────────────────
|
|
620
|
-
// EVERYTHING HOOKS
|
|
621
|
-
// ─────────────────────────────────────────────────────────
|
|
622
|
-
if (type === "everything") {
|
|
623
|
-
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
624
|
-
let events = record;
|
|
625
|
-
if (!events.length) return record;
|
|
626
|
-
const profile = meta && meta.profile ? meta.profile : {};
|
|
627
|
-
|
|
628
|
-
// Stamp superProps from profile for consistency
|
|
629
|
-
events.forEach(e => {
|
|
630
|
-
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
631
|
-
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
632
|
-
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
633
|
-
});
|
|
634
|
-
|
|
635
|
-
// ─────────────────────────────────────────────────────
|
|
636
|
-
// Hook #6: OUTAGE DAY (days 40-41)
|
|
637
|
-
// 40% of api calls get is_error=true with service errors
|
|
638
|
-
// ─────────────────────────────────────────────────────
|
|
639
|
-
events.forEach(e => {
|
|
640
|
-
if (e.event !== "api call") return;
|
|
641
|
-
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
642
|
-
if (dayInDataset >= 40 && dayInDataset < 42) {
|
|
643
|
-
if (chance.bool({ likelihood: 40 })) {
|
|
644
|
-
e.is_error = true;
|
|
645
|
-
e.error_type = chance.pickone([
|
|
646
|
-
"service_overloaded",
|
|
647
|
-
"internal_server_error",
|
|
648
|
-
"gateway_timeout",
|
|
649
|
-
]);
|
|
650
|
-
e.latency_ms = Math.floor((e.latency_ms || 1500) * 3);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
});
|
|
654
|
-
|
|
655
|
-
// Determine first event time for relative day calculations
|
|
656
|
-
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
657
|
-
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
658
|
-
|
|
659
|
-
// ─────────────────────────────────────────────────────
|
|
660
|
-
// Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
|
|
661
|
-
// Scale context_window and input_tokens by tier
|
|
662
|
-
// ─────────────────────────────────────────────────────
|
|
663
|
-
const tier = profile.api_tier || "Free";
|
|
664
|
-
const contextWindow = tier === "Enterprise" ? 2000000 : tier === "Build" ? 1000000 : 200000;
|
|
665
|
-
const inputMultiplier = tier === "Enterprise" ? 4 : tier === "Build" ? 2 : 1;
|
|
666
|
-
|
|
667
|
-
events.forEach(e => {
|
|
668
|
-
if (e.event === "api call") {
|
|
669
|
-
e.context_window = contextWindow;
|
|
670
|
-
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
671
|
-
}
|
|
672
|
-
});
|
|
673
|
-
|
|
674
|
-
// ─────────────────────────────────────────────────────
|
|
675
|
-
// Hook #1: PROMPT CACHING ADOPTION (CONVERSION)
|
|
676
|
-
// Users with any cache_enabled=true get 70% cost reduction
|
|
677
|
-
// on all subsequent api calls
|
|
678
|
-
// ─────────────────────────────────────────────────────
|
|
679
|
-
// ~25% of users have caching enabled on at least one event
|
|
680
|
-
const userId = events[0] && events[0].user_id;
|
|
681
|
-
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
682
|
-
const isCacheUser = (idHash % 4) === 0;
|
|
683
|
-
|
|
684
|
-
if (isCacheUser) {
|
|
685
|
-
let cacheActivated = false;
|
|
686
|
-
// Activate caching on events after the first 20% of user events
|
|
687
|
-
const activationPoint = Math.floor(events.length * 0.2);
|
|
688
|
-
events.forEach((e, idx) => {
|
|
689
|
-
if (e.event === "api call") {
|
|
690
|
-
if (idx >= activationPoint) {
|
|
691
|
-
cacheActivated = true;
|
|
692
|
-
}
|
|
693
|
-
if (cacheActivated) {
|
|
694
|
-
e.cache_enabled = true;
|
|
695
|
-
e.cost_usd = Math.round((e.cost_usd || 0.01) * 0.3 * 10000) / 10000;
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
});
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
// ─────────────────────────────────────────────────────
|
|
702
|
-
// Hook #2: MODEL MIGRATION WAVE
|
|
703
|
-
// After day 60, 35% of Build/Enterprise api_calls switch to
|
|
704
|
-
// opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
|
|
705
|
-
// (authoritative) and uses post-shift event timestamps.
|
|
706
|
-
// ─────────────────────────────────────────────────────
|
|
707
|
-
const datasetStartDay60 = datasetStart.add(60, "days");
|
|
708
|
-
if (tier === "Build" || tier === "Enterprise") {
|
|
709
|
-
events.forEach(e => {
|
|
710
|
-
if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
|
|
711
|
-
if (chance.bool({ likelihood: 35 })) {
|
|
712
|
-
e.model = "opus-4-7";
|
|
713
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
});
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
// ─────────────────────────────────────────────────────
|
|
720
|
-
// Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
|
|
721
|
-
// Users with 3+ tool use calls + 3+ multi_turn api calls
|
|
722
|
-
// get 8x tokens, 3x events. Threshold ensures a meaningful
|
|
723
|
-
// agentic cohort (~20-30% of users).
|
|
724
|
-
// ─────────────────────────────────────────────────────
|
|
725
|
-
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
726
|
-
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
727
|
-
const isAgenticUser = toolUseCount >= 3 && multiTurnCount >= 3;
|
|
728
|
-
|
|
729
|
-
if (isAgenticUser) {
|
|
730
|
-
events.forEach(e => {
|
|
731
|
-
if (e.event === "api call") {
|
|
732
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
|
|
733
|
-
}
|
|
734
|
-
});
|
|
735
|
-
|
|
736
|
-
const apiCalls = events.filter(e => e.event === "api call");
|
|
737
|
-
const extraCount = apiCalls.length * 2;
|
|
738
|
-
for (let i = 0; i < extraCount; i++) {
|
|
739
|
-
const template = apiCalls[i % apiCalls.length];
|
|
740
|
-
if (template) {
|
|
741
|
-
events.push({
|
|
742
|
-
...template,
|
|
743
|
-
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
744
|
-
user_id: template.user_id,
|
|
745
|
-
multi_turn: true,
|
|
746
|
-
});
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
// ─────────────────────────────────────────────────────
|
|
752
|
-
// Hook #4: RATE LIMIT CHURN
|
|
753
|
-
// >=2 rate limit errors in first 7 days -> remove 60% of
|
|
754
|
-
// events after week 1. Threshold is intentionally low because
|
|
755
|
-
// avgEventsPerUserPerDay=0.83 means most users only generate
|
|
756
|
-
// a handful of events per week.
|
|
757
|
-
// ─────────────────────────────────────────────────────
|
|
758
|
-
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
759
|
-
const earlyRateLimits = events.filter(e =>
|
|
760
|
-
e.event === "rate limit error" &&
|
|
761
|
-
dayjs(e.time).isBefore(firstWeekEnd)
|
|
762
|
-
).length;
|
|
763
|
-
|
|
764
|
-
if (earlyRateLimits >= 2) {
|
|
765
|
-
// Remove 60% of events after week 1
|
|
766
|
-
events = events.filter(e => {
|
|
767
|
-
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
768
|
-
return chance.bool({ likelihood: 40 });
|
|
769
|
-
}
|
|
770
|
-
return true;
|
|
771
|
-
});
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
// ─────────────────────────────────────────────────────
|
|
775
|
-
// Hook #7: BATCH API DISCOUNT (PURCHASE VALUE)
|
|
776
|
-
// Batch users get 50% lower cost_per_token, 2x tokens_used
|
|
777
|
-
// ─────────────────────────────────────────────────────
|
|
778
|
-
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
779
|
-
|
|
780
|
-
if (isBatchUser) {
|
|
781
|
-
events.forEach(e => {
|
|
782
|
-
if (e.event === "api call") {
|
|
783
|
-
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
|
|
784
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
|
|
785
|
-
}
|
|
786
|
-
});
|
|
787
|
-
}
|
|
788
|
-
|
|
789
|
-
// ─────────────────────────────────────────────────────
|
|
790
|
-
// Hook #8: EVAL-DRIVEN RETENTION
|
|
791
|
-
// Early eval users (first 7 days) get 75% D30 retention
|
|
792
|
-
// Non-eval users get only 25% D30 retention (remove events)
|
|
793
|
-
// ─────────────────────────────────────────────────────
|
|
794
|
-
const hasEarlyEval = events.some(e =>
|
|
795
|
-
e.event === "eval job" &&
|
|
796
|
-
dayjs(e.time).isBefore(firstWeekEnd)
|
|
797
|
-
);
|
|
798
|
-
|
|
799
|
-
if (hasEarlyEval) {
|
|
800
|
-
// Early eval users keep all their events (high retention)
|
|
801
|
-
} else {
|
|
802
|
-
// Non-eval users: remove 75% of events after day 30
|
|
803
|
-
const day30 = firstEventTime.add(30, "days");
|
|
804
|
-
events = events.filter(e => {
|
|
805
|
-
if (dayjs(e.time).isAfter(day30)) {
|
|
806
|
-
return chance.bool({ likelihood: 25 });
|
|
807
|
-
}
|
|
808
|
-
return true;
|
|
809
|
-
});
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
// ─────────────────────────────────────────────────────
|
|
813
|
-
// Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
|
|
814
|
-
// Count "docs searched" events between first "organization
|
|
815
|
-
// created" (sign-up) and any "billing payment". Sweet 2-4 → +35%
|
|
816
|
-
// on amount_usd of billing-payment events. Over 5+ → drop 30%
|
|
817
|
-
// of billing-payment events.
|
|
818
|
-
// ─────────────────────────────────────────────────────
|
|
819
|
-
const orgEvent = events.find(e => e.event === "organization created");
|
|
820
|
-
const firstBilling = events.find(e => e.event === "billing payment");
|
|
821
|
-
if (orgEvent && firstBilling) {
|
|
822
|
-
const aTime = dayjs(orgEvent.time);
|
|
823
|
-
const bTime = dayjs(firstBilling.time);
|
|
824
|
-
const docsBetween = events.filter(e =>
|
|
825
|
-
e.event === "docs searched" &&
|
|
826
|
-
dayjs(e.time).isAfter(aTime) &&
|
|
827
|
-
dayjs(e.time).isBefore(bTime)
|
|
828
|
-
).length;
|
|
829
|
-
if (docsBetween >= 2 && docsBetween <= 4) {
|
|
830
|
-
events.forEach(e => {
|
|
831
|
-
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
832
|
-
e.amount_usd = Math.round(e.amount_usd * 1.35);
|
|
833
|
-
}
|
|
834
|
-
});
|
|
835
|
-
} else if (docsBetween >= 5) {
|
|
836
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
837
|
-
if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
|
|
838
|
-
events.splice(i, 1);
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
return events;
|
|
845
|
-
}
|
|
846
|
-
|
|
811
|
+
hook(record, type, meta) {
|
|
812
|
+
if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
|
|
813
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
847
814
|
return record;
|
|
848
815
|
},
|
|
849
816
|
};
|