@ak--47/dungeon-master 1.5.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/create-dungeon/SKILL.md +139 -46
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +31 -6
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +44 -25
- package/.claude/skills/write-hooks/SKILL.md +31 -3
- package/CHANGELOG.md +85 -0
- package/HOOKS.md +13 -0
- package/dungeons/technical/ad-spend.js +41 -49
- package/dungeons/technical/anonymous-users.js +38 -36
- package/dungeons/technical/array-of-object-lookup.js +136 -153
- package/dungeons/technical/datagen-v15-verify.js +24 -11
- package/dungeons/technical/experiments.js +42 -40
- package/dungeons/technical/foobar.js +114 -118
- package/dungeons/technical/group-analytics.js +42 -40
- package/dungeons/technical/hook-helpers-verify.js +69 -50
- package/dungeons/technical/identity-model-verify.js +22 -12
- package/dungeons/technical/mirror-strategies.js +37 -39
- package/dungeons/technical/nested-objects.js +119 -118
- package/dungeons/technical/pattern-aggregate-by-bin.js +21 -8
- package/dungeons/technical/pattern-attributed-by-source.js +23 -9
- package/dungeons/technical/pattern-frequency-by-frequency.js +21 -8
- package/dungeons/technical/pattern-funnel-frequency.js +30 -15
- package/dungeons/technical/pattern-ttc-by-segment.js +21 -8
- package/dungeons/technical/retention-cadence.js +115 -112
- package/dungeons/technical/sanity.js +86 -80
- package/dungeons/technical/scale-test.js +34 -38
- package/dungeons/technical/scd.js +111 -128
- package/dungeons/technical/simple.js +134 -141
- package/dungeons/technical/simplest.js +54 -62
- package/dungeons/technical/text-generation.js +110 -146
- package/dungeons/vertical/ai-platform.js +296 -333
- package/dungeons/vertical/community.js +284 -255
- package/dungeons/vertical/crypto.js +395 -391
- package/dungeons/vertical/dating.js +411 -378
- package/dungeons/vertical/devtools.js +336 -298
- package/dungeons/vertical/ecommerce.js +316 -394
- package/dungeons/vertical/education.js +369 -325
- package/dungeons/vertical/fintech.js +358 -325
- package/dungeons/vertical/fitness.js +335 -291
- package/dungeons/vertical/food-delivery.js +343 -307
- package/dungeons/vertical/gaming.js +480 -444
- package/dungeons/vertical/healthcare.js +306 -262
- package/dungeons/vertical/insurance-application.js +427 -409
- package/dungeons/vertical/logistics.js +271 -252
- package/dungeons/vertical/marketplace.js +333 -323
- package/dungeons/vertical/media.js +382 -335
- package/dungeons/vertical/real-estate.js +395 -346
- package/dungeons/vertical/sass.js +319 -333
- package/dungeons/vertical/social.js +368 -316
- package/dungeons/vertical/travel.js +297 -295
- package/index.js +46 -4
- package/lib/core/config-validator.js +126 -28
- package/lib/generators/funnels.js +4 -1
- package/lib/orchestrators/mixpanel-sender.js +7 -0
- package/lib/orchestrators/user-loop.js +132 -31
- package/lib/templates/defaults.js +59 -59
- package/lib/templates/macro-presets.js +14 -2
- package/lib/utils/dataset-context.js +103 -0
- package/lib/utils/retention-curve.js +140 -0
- package/lib/utils/utils.js +149 -38
- package/lib/verify/counting.js +40 -0
- package/lib/verify/emulate-breakdown.js +20 -1
- package/lib/verify/index.js +1 -0
- package/lib/verify/schema-validator.js +3 -1
- package/package.json +11 -2
- package/scripts/run-dungeon.mjs +12 -1
- package/types.d.ts +117 -1
|
@@ -1,56 +1,43 @@
|
|
|
1
|
-
// ──
|
|
2
|
-
const SEED = "promptforge";
|
|
3
|
-
const num_days = 120;
|
|
4
|
-
const num_users = 10_000;
|
|
5
|
-
const avg_events_per_user_per_day = 0.83;
|
|
6
|
-
let token = "your-mixpanel-token";
|
|
7
|
-
|
|
8
|
-
// ── env overrides ──
|
|
9
|
-
if (process.env.MP_TOKEN) token = process.env.MP_TOKEN;
|
|
10
|
-
|
|
1
|
+
// ── IMPORTS ──
|
|
11
2
|
import dayjs from "dayjs";
|
|
12
3
|
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
13
5
|
import "dotenv/config";
|
|
14
6
|
import * as u from "../../lib/utils/utils.js";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
7
|
+
/** @typedef {import("../../types").Dungeon} Config */
|
|
8
|
+
|
|
9
|
+
// ── OVERVIEW ──
|
|
10
|
+
/*
|
|
11
|
+
* NAME: PromptForge
|
|
12
|
+
* APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
|
|
13
|
+
* requests for chat completions, embeddings, evaluations, and tool
|
|
14
|
+
* use. Billing is per input/output token. Features: prompt caching,
|
|
15
|
+
* tool use, multi-turn conversations, batch API, model selection,
|
|
16
|
+
* eval pipelines.
|
|
17
|
+
* SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
|
|
18
|
+
* CORE LOOP: organization created → api key created → api call → iterate
|
|
19
|
+
*
|
|
20
|
+
* EVENTS (18):
|
|
21
|
+
* api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
|
|
22
|
+
* > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
|
|
23
|
+
* > model selected (3) > api key created (2) > batch job submitted (2)
|
|
24
|
+
* > batch job completed (2) > billing payment (2) > member invited (2)
|
|
25
|
+
* > organization created (1) > api key rotated (1) > webhook configured (1)
|
|
26
|
+
* > account deactivated (1)
|
|
27
|
+
*
|
|
28
|
+
* FUNNELS (3):
|
|
29
|
+
* - Onboarding: organization created → api key created → api call (70%)
|
|
30
|
+
* - API to Eval Pipeline: api call → tool use call → eval job (45%)
|
|
31
|
+
* - Usage to Billing: api call → billing payment (30%)
|
|
32
|
+
*
|
|
33
|
+
* USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
|
|
34
|
+
* SUPER PROPS: api_tier, primary_use_case, sdk_language
|
|
35
|
+
* SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
|
|
36
|
+
* GROUPS: none
|
|
37
|
+
*/
|
|
20
38
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
* DATASET OVERVIEW
|
|
24
|
-
* ===============================================================
|
|
25
|
-
*
|
|
26
|
-
* PromptForge -- an LLM API platform (like Anthropic/OpenAI).
|
|
27
|
-
* Customers (developers and companies) send API requests for chat
|
|
28
|
-
* completions, embeddings, evaluations, and tool use. Billing is
|
|
29
|
-
* per input/output token. Key features: prompt caching, tool use,
|
|
30
|
-
* multi-turn conversations, batch API, model selection, and
|
|
31
|
-
* evaluation pipelines.
|
|
32
|
-
*
|
|
33
|
-
* - 8,000 users over 120 days, ~800K events
|
|
34
|
-
* - Three API tiers: Free, Build, Enterprise
|
|
35
|
-
* - Core loop: org created -> api key created -> api call -> iterate
|
|
36
|
-
* - Revenue: token-based billing with tier-based pricing
|
|
37
|
-
*
|
|
38
|
-
* Key entities:
|
|
39
|
-
* - model: LLM model version (sonnet-4, haiku-4, opus-4-6, opus-4-7)
|
|
40
|
-
* - api_tier: Free / Build / Enterprise (determines context window, rate limits)
|
|
41
|
-
* - tokens_used: total tokens consumed per API call (input + output)
|
|
42
|
-
* - cost_usd: dollar cost of a single API call
|
|
43
|
-
* - cache_enabled: prompt caching flag that reduces cost 70%
|
|
44
|
-
* - multi_turn: whether the call is part of a conversation
|
|
45
|
-
*
|
|
46
|
-
* ===============================================================
|
|
47
|
-
* ANALYTICS HOOKS (10 hooks)
|
|
48
|
-
* ===============================================================
|
|
49
|
-
*
|
|
50
|
-
* NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
|
|
51
|
-
* only via behavioral cohorts (count event per user) or raw-prop
|
|
52
|
-
* breakdowns (api_tier from profile, model, error_type).
|
|
53
|
-
*
|
|
39
|
+
// ── HOOK STORIES ──
|
|
40
|
+
/*
|
|
54
41
|
* ---------------------------------------------------------------
|
|
55
42
|
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
56
43
|
* ---------------------------------------------------------------
|
|
@@ -315,36 +302,273 @@ const chance = u.initChance(SEED);
|
|
|
315
302
|
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
316
303
|
*/
|
|
317
304
|
|
|
305
|
+
// ── SCALE ──
|
|
306
|
+
const SEED = "promptforge";
|
|
307
|
+
const NUM_USERS = 10_000;
|
|
308
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
309
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
310
|
+
const EVENTS_PER_DAY = 0.83;
|
|
311
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
312
|
+
|
|
313
|
+
const chance = u.initChance(SEED);
|
|
314
|
+
|
|
315
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
316
|
+
const OUTAGE_START_DAY = 40;
|
|
317
|
+
const OUTAGE_END_DAY = 42;
|
|
318
|
+
const OUTAGE_ERROR_LIKELIHOOD = 40;
|
|
319
|
+
const OUTAGE_LATENCY_MULT = 3;
|
|
320
|
+
|
|
321
|
+
const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
|
|
322
|
+
const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
|
|
323
|
+
|
|
324
|
+
const CACHE_USER_HASH_MOD = 4;
|
|
325
|
+
const CACHE_COST_FACTOR = 0.3;
|
|
326
|
+
const CACHE_ACTIVATION_PCT = 0.2;
|
|
327
|
+
|
|
328
|
+
const MODEL_MIGRATION_DAY = 60;
|
|
329
|
+
const MODEL_MIGRATION_LIKELIHOOD = 35;
|
|
330
|
+
const MODEL_MIGRATION_TOKEN_MULT = 1.5;
|
|
331
|
+
|
|
332
|
+
const AGENTIC_TOOL_THRESHOLD = 3;
|
|
333
|
+
const AGENTIC_MULTITURN_THRESHOLD = 3;
|
|
334
|
+
const AGENTIC_TOKEN_MULT = 8;
|
|
335
|
+
const AGENTIC_CLONE_MULT = 2;
|
|
336
|
+
|
|
337
|
+
const RATE_LIMIT_THRESHOLD = 2;
|
|
338
|
+
const RATE_LIMIT_KEEP_LIKELIHOOD = 40;
|
|
339
|
+
|
|
340
|
+
const BATCH_COST_FACTOR = 0.5;
|
|
341
|
+
const BATCH_TOKEN_MULT = 2;
|
|
342
|
+
|
|
343
|
+
const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
|
|
344
|
+
const EVAL_CUTOFF_DAYS = 30;
|
|
345
|
+
|
|
346
|
+
const DOCS_SWEET_MIN = 2;
|
|
347
|
+
const DOCS_SWEET_MAX = 4;
|
|
348
|
+
const DOCS_OVER_THRESHOLD = 5;
|
|
349
|
+
const DOCS_BILLING_BOOST = 1.35;
|
|
350
|
+
const DOCS_BILLING_DROP_LIKELIHOOD = 30;
|
|
351
|
+
|
|
352
|
+
const FUNNEL_TTC_ENTERPRISE = 0.67;
|
|
353
|
+
const FUNNEL_TTC_FREE = 1.4;
|
|
354
|
+
|
|
355
|
+
// ── HELPER FUNCTIONS ──
|
|
356
|
+
function handleFunnelPostHooks(record, meta) {
|
|
357
|
+
// H9: API-to-Eval TTC scaled by tier
|
|
358
|
+
const segment = meta?.profile?.api_tier;
|
|
359
|
+
if (Array.isArray(record) && record.length > 1) {
|
|
360
|
+
const factor = (
|
|
361
|
+
segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
|
|
362
|
+
segment === "Free" ? FUNNEL_TTC_FREE :
|
|
363
|
+
1.0
|
|
364
|
+
);
|
|
365
|
+
if (factor !== 1.0) {
|
|
366
|
+
for (let i = 1; i < record.length; i++) {
|
|
367
|
+
const prev = dayjs(record[i - 1].time);
|
|
368
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
369
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return record;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function handleEverythingHooks(record, meta) {
|
|
377
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
378
|
+
let events = record;
|
|
379
|
+
if (!events.length) return record;
|
|
380
|
+
const profile = meta && meta.profile ? meta.profile : {};
|
|
381
|
+
|
|
382
|
+
events.forEach(e => {
|
|
383
|
+
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
384
|
+
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
385
|
+
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// H6: Outage day (days 40-41) — 40% of api calls flipped to errors
|
|
389
|
+
events.forEach(e => {
|
|
390
|
+
if (e.event !== "api call") return;
|
|
391
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
392
|
+
if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
|
|
393
|
+
if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
|
|
394
|
+
e.is_error = true;
|
|
395
|
+
e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
|
|
396
|
+
e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
402
|
+
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
403
|
+
|
|
404
|
+
// H5: Tier-based context window & input tokens
|
|
405
|
+
const tier = profile.api_tier || "Free";
|
|
406
|
+
const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
|
|
407
|
+
const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
|
|
408
|
+
events.forEach(e => {
|
|
409
|
+
if (e.event === "api call") {
|
|
410
|
+
e.context_window = contextWindow;
|
|
411
|
+
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
// H1: Prompt caching adoption — ~25% of users; activates 20% into stream
|
|
416
|
+
const userId = events[0] && events[0].user_id;
|
|
417
|
+
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
418
|
+
const isCacheUser = (idHash % CACHE_USER_HASH_MOD) === 0;
|
|
419
|
+
if (isCacheUser) {
|
|
420
|
+
let cacheActivated = false;
|
|
421
|
+
const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
|
|
422
|
+
events.forEach((e, idx) => {
|
|
423
|
+
if (e.event === "api call") {
|
|
424
|
+
if (idx >= activationPoint) cacheActivated = true;
|
|
425
|
+
if (cacheActivated) {
|
|
426
|
+
e.cache_enabled = true;
|
|
427
|
+
e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// H2: Model migration wave — post-day-60 Build/Enterprise calls migrate to opus-4-7
|
|
434
|
+
const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
|
|
435
|
+
if (tier === "Build" || tier === "Enterprise") {
|
|
436
|
+
events.forEach(e => {
|
|
437
|
+
if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
|
|
438
|
+
if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
|
|
439
|
+
e.model = "opus-4-7";
|
|
440
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens, 3x events
|
|
447
|
+
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
448
|
+
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
449
|
+
const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
|
|
450
|
+
if (isAgenticUser) {
|
|
451
|
+
events.forEach(e => {
|
|
452
|
+
if (e.event === "api call") {
|
|
453
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
const apiCalls = events.filter(e => e.event === "api call");
|
|
457
|
+
const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
|
|
458
|
+
for (let i = 0; i < extraCount; i++) {
|
|
459
|
+
const template = apiCalls[i % apiCalls.length];
|
|
460
|
+
if (template) {
|
|
461
|
+
events.push({
|
|
462
|
+
...template,
|
|
463
|
+
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
464
|
+
user_id: template.user_id,
|
|
465
|
+
multi_turn: true,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// H4: Rate-limit churn — 2+ early rate-limit errors → drop 60% of post-week-1 events
|
|
472
|
+
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
473
|
+
const earlyRateLimits = events.filter(e =>
|
|
474
|
+
e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
475
|
+
).length;
|
|
476
|
+
if (earlyRateLimits >= RATE_LIMIT_THRESHOLD) {
|
|
477
|
+
events = events.filter(e => {
|
|
478
|
+
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
479
|
+
return chance.bool({ likelihood: RATE_LIMIT_KEEP_LIKELIHOOD });
|
|
480
|
+
}
|
|
481
|
+
return true;
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// H7: Batch API discount — any batch job submitted → 50% cost, 2x tokens
|
|
486
|
+
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
487
|
+
if (isBatchUser) {
|
|
488
|
+
events.forEach(e => {
|
|
489
|
+
if (e.event === "api call") {
|
|
490
|
+
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
|
|
491
|
+
e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// H8: Eval-driven retention — non-eval users lose 75% post-day-30
|
|
497
|
+
const hasEarlyEval = events.some(e =>
|
|
498
|
+
e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
499
|
+
);
|
|
500
|
+
if (!hasEarlyEval) {
|
|
501
|
+
const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
|
|
502
|
+
events = events.filter(e => {
|
|
503
|
+
if (dayjs(e.time).isAfter(cutoff)) {
|
|
504
|
+
return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
|
|
505
|
+
}
|
|
506
|
+
return true;
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// H10: Docs-searched magic number — count between org-created and first billing
|
|
511
|
+
const orgEvent = events.find(e => e.event === "organization created");
|
|
512
|
+
const firstBilling = events.find(e => e.event === "billing payment");
|
|
513
|
+
if (orgEvent && firstBilling) {
|
|
514
|
+
const aTime = dayjs(orgEvent.time);
|
|
515
|
+
const bTime = dayjs(firstBilling.time);
|
|
516
|
+
const docsBetween = events.filter(e =>
|
|
517
|
+
e.event === "docs searched" &&
|
|
518
|
+
dayjs(e.time).isAfter(aTime) &&
|
|
519
|
+
dayjs(e.time).isBefore(bTime)
|
|
520
|
+
).length;
|
|
521
|
+
if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
|
|
522
|
+
events.forEach(e => {
|
|
523
|
+
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
524
|
+
e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
} else if (docsBetween >= DOCS_OVER_THRESHOLD) {
|
|
528
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
529
|
+
if (events[i].event === "billing payment" && chance.bool({ likelihood: DOCS_BILLING_DROP_LIKELIHOOD })) {
|
|
530
|
+
events.splice(i, 1);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
return events;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// ── CONFIG ──
|
|
318
540
|
/** @type {Config} */
|
|
319
541
|
const config = {
|
|
320
542
|
version: 2,
|
|
321
|
-
token,
|
|
322
543
|
seed: SEED,
|
|
323
|
-
datasetStart:
|
|
324
|
-
datasetEnd:
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
numUsers: num_users,
|
|
328
|
-
hasAnonIds: true,
|
|
329
|
-
avgDevicePerUser: 2,
|
|
330
|
-
hasSessionIds: true,
|
|
544
|
+
datasetStart: DATASET_START,
|
|
545
|
+
datasetEnd: DATASET_END,
|
|
546
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
547
|
+
numUsers: NUM_USERS,
|
|
331
548
|
format: "json",
|
|
332
549
|
gzip: true,
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
550
|
+
credentials: {
|
|
551
|
+
token,
|
|
552
|
+
},
|
|
553
|
+
switches: {
|
|
554
|
+
hasSessionIds: true,
|
|
555
|
+
alsoInferFunnels: false,
|
|
556
|
+
hasLocation: true,
|
|
557
|
+
hasAndroidDevices: false,
|
|
558
|
+
hasIOSDevices: false,
|
|
559
|
+
hasDesktopDevices: true,
|
|
560
|
+
hasBrowser: true,
|
|
561
|
+
hasCampaigns: false,
|
|
562
|
+
isAnonymous: false,
|
|
563
|
+
hasAdSpend: false,
|
|
564
|
+
hasAvatar: true,
|
|
565
|
+
},
|
|
566
|
+
identity: {
|
|
567
|
+
avgDevicePerUser: 2,
|
|
568
|
+
},
|
|
343
569
|
concurrency: 1,
|
|
344
570
|
writeToDisk: false,
|
|
345
|
-
|
|
346
571
|
soup: "growth",
|
|
347
|
-
|
|
348
572
|
scdProps: {
|
|
349
573
|
monthly_api_usage: {
|
|
350
574
|
values: u.weighNumRange(0, 1000000, 0.3, 50),
|
|
@@ -359,8 +583,6 @@ const config = {
|
|
|
359
583
|
max: 6,
|
|
360
584
|
},
|
|
361
585
|
},
|
|
362
|
-
|
|
363
|
-
// -- Events (18) ------------------------------------------
|
|
364
586
|
events: [
|
|
365
587
|
{
|
|
366
588
|
event: "organization created",
|
|
@@ -546,8 +768,6 @@ const config = {
|
|
|
546
768
|
},
|
|
547
769
|
},
|
|
548
770
|
],
|
|
549
|
-
|
|
550
|
-
// -- Funnels (3) ------------------------------------------
|
|
551
771
|
funnels: [
|
|
552
772
|
{
|
|
553
773
|
name: "Onboarding",
|
|
@@ -575,15 +795,11 @@ const config = {
|
|
|
575
795
|
weight: 2,
|
|
576
796
|
},
|
|
577
797
|
],
|
|
578
|
-
|
|
579
|
-
// -- SuperProps --------------------------------------------
|
|
580
798
|
superProps: {
|
|
581
799
|
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
582
800
|
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
583
801
|
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
584
802
|
},
|
|
585
|
-
|
|
586
|
-
// -- UserProps ---------------------------------------------
|
|
587
803
|
userProps: {
|
|
588
804
|
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
589
805
|
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
@@ -592,262 +808,9 @@ const config = {
|
|
|
592
808
|
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
593
809
|
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
594
810
|
},
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
// ─────────────────────────────────────────────────────────
|
|
600
|
-
// Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
601
|
-
// Enterprise users complete API to Eval Pipeline funnel 1.5x faster
|
|
602
|
-
// (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
|
|
603
|
-
// 1.4). Mutates record[i].time. No flag.
|
|
604
|
-
// ─────────────────────────────────────────────────────────
|
|
605
|
-
if (type === "funnel-post") {
|
|
606
|
-
const segment = meta?.profile?.api_tier;
|
|
607
|
-
if (Array.isArray(record) && record.length > 1) {
|
|
608
|
-
const factor = (
|
|
609
|
-
segment === "Enterprise" ? 0.67 :
|
|
610
|
-
segment === "Free" ? 1.4 :
|
|
611
|
-
1.0
|
|
612
|
-
);
|
|
613
|
-
if (factor !== 1.0) {
|
|
614
|
-
for (let i = 1; i < record.length; i++) {
|
|
615
|
-
const prev = dayjs(record[i - 1].time);
|
|
616
|
-
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
617
|
-
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
// ─────────────────────────────────────────────────────────
|
|
624
|
-
// EVERYTHING HOOKS
|
|
625
|
-
// ─────────────────────────────────────────────────────────
|
|
626
|
-
if (type === "everything") {
|
|
627
|
-
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
628
|
-
let events = record;
|
|
629
|
-
if (!events.length) return record;
|
|
630
|
-
const profile = meta && meta.profile ? meta.profile : {};
|
|
631
|
-
|
|
632
|
-
// Stamp superProps from profile for consistency
|
|
633
|
-
events.forEach(e => {
|
|
634
|
-
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
635
|
-
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
636
|
-
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
637
|
-
});
|
|
638
|
-
|
|
639
|
-
// ─────────────────────────────────────────────────────
|
|
640
|
-
// Hook #6: OUTAGE DAY (days 40-41)
|
|
641
|
-
// 40% of api calls get is_error=true with service errors
|
|
642
|
-
// ─────────────────────────────────────────────────────
|
|
643
|
-
events.forEach(e => {
|
|
644
|
-
if (e.event !== "api call") return;
|
|
645
|
-
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
646
|
-
if (dayInDataset >= 40 && dayInDataset < 42) {
|
|
647
|
-
if (chance.bool({ likelihood: 40 })) {
|
|
648
|
-
e.is_error = true;
|
|
649
|
-
e.error_type = chance.pickone([
|
|
650
|
-
"service_overloaded",
|
|
651
|
-
"internal_server_error",
|
|
652
|
-
"gateway_timeout",
|
|
653
|
-
]);
|
|
654
|
-
e.latency_ms = Math.floor((e.latency_ms || 1500) * 3);
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
});
|
|
658
|
-
|
|
659
|
-
// Determine first event time for relative day calculations
|
|
660
|
-
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
661
|
-
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
662
|
-
|
|
663
|
-
// ─────────────────────────────────────────────────────
|
|
664
|
-
// Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
|
|
665
|
-
// Scale context_window and input_tokens by tier
|
|
666
|
-
// ─────────────────────────────────────────────────────
|
|
667
|
-
const tier = profile.api_tier || "Free";
|
|
668
|
-
const contextWindow = tier === "Enterprise" ? 2000000 : tier === "Build" ? 1000000 : 200000;
|
|
669
|
-
const inputMultiplier = tier === "Enterprise" ? 4 : tier === "Build" ? 2 : 1;
|
|
670
|
-
|
|
671
|
-
events.forEach(e => {
|
|
672
|
-
if (e.event === "api call") {
|
|
673
|
-
e.context_window = contextWindow;
|
|
674
|
-
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
675
|
-
}
|
|
676
|
-
});
|
|
677
|
-
|
|
678
|
-
// ─────────────────────────────────────────────────────
|
|
679
|
-
// Hook #1: PROMPT CACHING ADOPTION (CONVERSION)
|
|
680
|
-
// Users with any cache_enabled=true get 70% cost reduction
|
|
681
|
-
// on all subsequent api calls
|
|
682
|
-
// ─────────────────────────────────────────────────────
|
|
683
|
-
// ~25% of users have caching enabled on at least one event
|
|
684
|
-
const userId = events[0] && events[0].user_id;
|
|
685
|
-
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
686
|
-
const isCacheUser = (idHash % 4) === 0;
|
|
687
|
-
|
|
688
|
-
if (isCacheUser) {
|
|
689
|
-
let cacheActivated = false;
|
|
690
|
-
// Activate caching on events after the first 20% of user events
|
|
691
|
-
const activationPoint = Math.floor(events.length * 0.2);
|
|
692
|
-
events.forEach((e, idx) => {
|
|
693
|
-
if (e.event === "api call") {
|
|
694
|
-
if (idx >= activationPoint) {
|
|
695
|
-
cacheActivated = true;
|
|
696
|
-
}
|
|
697
|
-
if (cacheActivated) {
|
|
698
|
-
e.cache_enabled = true;
|
|
699
|
-
e.cost_usd = Math.round((e.cost_usd || 0.01) * 0.3 * 10000) / 10000;
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
});
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
// ─────────────────────────────────────────────────────
|
|
706
|
-
// Hook #2: MODEL MIGRATION WAVE
|
|
707
|
-
// After day 60, 35% of Build/Enterprise api_calls switch to
|
|
708
|
-
// opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
|
|
709
|
-
// (authoritative) and uses post-shift event timestamps.
|
|
710
|
-
// ─────────────────────────────────────────────────────
|
|
711
|
-
const datasetStartDay60 = datasetStart.add(60, "days");
|
|
712
|
-
if (tier === "Build" || tier === "Enterprise") {
|
|
713
|
-
events.forEach(e => {
|
|
714
|
-
if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
|
|
715
|
-
if (chance.bool({ likelihood: 35 })) {
|
|
716
|
-
e.model = "opus-4-7";
|
|
717
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
});
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
// ─────────────────────────────────────────────────────
|
|
724
|
-
// Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
|
|
725
|
-
// Users with 3+ tool use calls + 3+ multi_turn api calls
|
|
726
|
-
// get 8x tokens, 3x events. Threshold ensures a meaningful
|
|
727
|
-
// agentic cohort (~20-30% of users).
|
|
728
|
-
// ─────────────────────────────────────────────────────
|
|
729
|
-
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
730
|
-
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
731
|
-
const isAgenticUser = toolUseCount >= 3 && multiTurnCount >= 3;
|
|
732
|
-
|
|
733
|
-
if (isAgenticUser) {
|
|
734
|
-
events.forEach(e => {
|
|
735
|
-
if (e.event === "api call") {
|
|
736
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
|
|
737
|
-
}
|
|
738
|
-
});
|
|
739
|
-
|
|
740
|
-
const apiCalls = events.filter(e => e.event === "api call");
|
|
741
|
-
const extraCount = apiCalls.length * 2;
|
|
742
|
-
for (let i = 0; i < extraCount; i++) {
|
|
743
|
-
const template = apiCalls[i % apiCalls.length];
|
|
744
|
-
if (template) {
|
|
745
|
-
events.push({
|
|
746
|
-
...template,
|
|
747
|
-
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
748
|
-
user_id: template.user_id,
|
|
749
|
-
multi_turn: true,
|
|
750
|
-
});
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
// ─────────────────────────────────────────────────────
|
|
756
|
-
// Hook #4: RATE LIMIT CHURN
|
|
757
|
-
// >=2 rate limit errors in first 7 days -> remove 60% of
|
|
758
|
-
// events after week 1. Threshold is intentionally low because
|
|
759
|
-
// avgEventsPerUserPerDay=0.83 means most users only generate
|
|
760
|
-
// a handful of events per week.
|
|
761
|
-
// ─────────────────────────────────────────────────────
|
|
762
|
-
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
763
|
-
const earlyRateLimits = events.filter(e =>
|
|
764
|
-
e.event === "rate limit error" &&
|
|
765
|
-
dayjs(e.time).isBefore(firstWeekEnd)
|
|
766
|
-
).length;
|
|
767
|
-
|
|
768
|
-
if (earlyRateLimits >= 2) {
|
|
769
|
-
// Remove 60% of events after week 1
|
|
770
|
-
events = events.filter(e => {
|
|
771
|
-
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
772
|
-
return chance.bool({ likelihood: 40 });
|
|
773
|
-
}
|
|
774
|
-
return true;
|
|
775
|
-
});
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
// ─────────────────────────────────────────────────────
|
|
779
|
-
// Hook #7: BATCH API DISCOUNT (PURCHASE VALUE)
|
|
780
|
-
// Batch users get 50% lower cost_per_token, 2x tokens_used
|
|
781
|
-
// ─────────────────────────────────────────────────────
|
|
782
|
-
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
783
|
-
|
|
784
|
-
if (isBatchUser) {
|
|
785
|
-
events.forEach(e => {
|
|
786
|
-
if (e.event === "api call") {
|
|
787
|
-
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
|
|
788
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
|
|
789
|
-
}
|
|
790
|
-
});
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
// ─────────────────────────────────────────────────────
|
|
794
|
-
// Hook #8: EVAL-DRIVEN RETENTION
|
|
795
|
-
// Early eval users (first 7 days) get 75% D30 retention
|
|
796
|
-
// Non-eval users get only 25% D30 retention (remove events)
|
|
797
|
-
// ─────────────────────────────────────────────────────
|
|
798
|
-
const hasEarlyEval = events.some(e =>
|
|
799
|
-
e.event === "eval job" &&
|
|
800
|
-
dayjs(e.time).isBefore(firstWeekEnd)
|
|
801
|
-
);
|
|
802
|
-
|
|
803
|
-
if (hasEarlyEval) {
|
|
804
|
-
// Early eval users keep all their events (high retention)
|
|
805
|
-
} else {
|
|
806
|
-
// Non-eval users: remove 75% of events after day 30
|
|
807
|
-
const day30 = firstEventTime.add(30, "days");
|
|
808
|
-
events = events.filter(e => {
|
|
809
|
-
if (dayjs(e.time).isAfter(day30)) {
|
|
810
|
-
return chance.bool({ likelihood: 25 });
|
|
811
|
-
}
|
|
812
|
-
return true;
|
|
813
|
-
});
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
// ─────────────────────────────────────────────────────
|
|
817
|
-
// Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
|
|
818
|
-
// Count "docs searched" events between first "organization
|
|
819
|
-
// created" (sign-up) and any "billing payment". Sweet 2-4 → +35%
|
|
820
|
-
// on amount_usd of billing-payment events. Over 5+ → drop 30%
|
|
821
|
-
// of billing-payment events.
|
|
822
|
-
// ─────────────────────────────────────────────────────
|
|
823
|
-
const orgEvent = events.find(e => e.event === "organization created");
|
|
824
|
-
const firstBilling = events.find(e => e.event === "billing payment");
|
|
825
|
-
if (orgEvent && firstBilling) {
|
|
826
|
-
const aTime = dayjs(orgEvent.time);
|
|
827
|
-
const bTime = dayjs(firstBilling.time);
|
|
828
|
-
const docsBetween = events.filter(e =>
|
|
829
|
-
e.event === "docs searched" &&
|
|
830
|
-
dayjs(e.time).isAfter(aTime) &&
|
|
831
|
-
dayjs(e.time).isBefore(bTime)
|
|
832
|
-
).length;
|
|
833
|
-
if (docsBetween >= 2 && docsBetween <= 4) {
|
|
834
|
-
events.forEach(e => {
|
|
835
|
-
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
836
|
-
e.amount_usd = Math.round(e.amount_usd * 1.35);
|
|
837
|
-
}
|
|
838
|
-
});
|
|
839
|
-
} else if (docsBetween >= 5) {
|
|
840
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
841
|
-
if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
|
|
842
|
-
events.splice(i, 1);
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
}
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
return events;
|
|
849
|
-
}
|
|
850
|
-
|
|
811
|
+
hook(record, type, meta) {
|
|
812
|
+
if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
|
|
813
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
851
814
|
return record;
|
|
852
815
|
},
|
|
853
816
|
};
|