@ak--47/dungeon-master 1.4.5 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +158 -0
  2. package/.claude/skills/create-dungeon/SKILL.md +464 -0
  3. package/.claude/skills/verify-dungeon/SKILL.md +157 -0
  4. package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
  5. package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
  6. package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
  7. package/.claude/skills/write-hooks/SKILL.md +468 -0
  8. package/CHANGELOG.md +182 -0
  9. package/HOOKS.md +1256 -597
  10. package/README.md +140 -5
  11. package/dungeons/technical/ad-spend.js +41 -49
  12. package/dungeons/technical/anonymous-users.js +38 -36
  13. package/dungeons/technical/array-of-object-lookup.js +136 -153
  14. package/dungeons/technical/datagen-v15-verify.js +87 -0
  15. package/dungeons/technical/experiments.js +42 -40
  16. package/dungeons/technical/foobar.js +114 -118
  17. package/dungeons/technical/group-analytics.js +42 -40
  18. package/dungeons/technical/hook-helpers-verify.js +69 -50
  19. package/dungeons/technical/identity-model-verify.js +22 -12
  20. package/dungeons/technical/mirror-strategies.js +37 -39
  21. package/dungeons/technical/nested-objects.js +119 -118
  22. package/dungeons/technical/pattern-aggregate-by-bin.js +21 -8
  23. package/dungeons/technical/pattern-attributed-by-source.js +23 -9
  24. package/dungeons/technical/pattern-frequency-by-frequency.js +21 -8
  25. package/dungeons/technical/pattern-funnel-frequency.js +30 -15
  26. package/dungeons/technical/pattern-ttc-by-segment.js +21 -8
  27. package/dungeons/technical/retention-cadence.js +115 -112
  28. package/dungeons/technical/sanity.js +86 -80
  29. package/dungeons/technical/scale-test.js +34 -38
  30. package/dungeons/technical/scd.js +111 -128
  31. package/dungeons/technical/simple.js +134 -141
  32. package/dungeons/technical/simplest.js +111 -65
  33. package/dungeons/technical/text-generation.js +110 -146
  34. package/dungeons/vertical/ai-platform.js +300 -333
  35. package/dungeons/vertical/community.js +290 -255
  36. package/dungeons/vertical/crypto.js +400 -391
  37. package/dungeons/vertical/dating.js +421 -375
  38. package/dungeons/vertical/devtools.js +346 -298
  39. package/dungeons/vertical/ecommerce.js +322 -394
  40. package/dungeons/vertical/education.js +380 -325
  41. package/dungeons/vertical/fintech.js +371 -325
  42. package/dungeons/vertical/fitness.js +345 -291
  43. package/dungeons/vertical/food-delivery.js +352 -307
  44. package/dungeons/vertical/gaming.js +490 -444
  45. package/dungeons/vertical/healthcare.js +311 -262
  46. package/dungeons/vertical/insurance-application.js +437 -409
  47. package/dungeons/vertical/logistics.js +278 -252
  48. package/dungeons/vertical/marketplace.js +340 -323
  49. package/dungeons/vertical/media.js +390 -335
  50. package/dungeons/vertical/real-estate.js +402 -347
  51. package/dungeons/vertical/sass.js +331 -333
  52. package/dungeons/vertical/social.js +377 -316
  53. package/dungeons/vertical/travel.js +302 -295
  54. package/index.js +64 -7
  55. package/lib/core/config-validator.js +378 -17
  56. package/lib/core/dungeon-loader.js +2 -5
  57. package/lib/generators/events.js +12 -13
  58. package/lib/generators/funnels.js +76 -2
  59. package/lib/hook-helpers/index.js +1 -0
  60. package/lib/hook-helpers/inject.js +95 -0
  61. package/lib/orchestrators/mixpanel-sender.js +7 -0
  62. package/lib/orchestrators/user-loop.js +598 -48
  63. package/lib/templates/defaults.js +59 -59
  64. package/lib/templates/macro-presets.js +53 -11
  65. package/lib/utils/dataset-context.js +103 -0
  66. package/lib/utils/retention-curve.js +140 -0
  67. package/lib/utils/utils.js +157 -109
  68. package/lib/verify/counting.js +360 -0
  69. package/lib/verify/emulate-breakdown.js +531 -108
  70. package/lib/verify/funnel-engine.js +539 -0
  71. package/lib/verify/identity.js +78 -0
  72. package/lib/verify/index.js +20 -0
  73. package/lib/verify/schema-validator.js +3 -1
  74. package/lib/verify/verify-dungeon.js +58 -0
  75. package/package.json +14 -3
  76. package/scripts/run-dungeon.mjs +12 -1
  77. package/types.d.ts +353 -4
  78. package/scripts/smoke-test-all.mjs +0 -162
@@ -1,56 +1,43 @@
1
- // ── TWEAK THESE ──
2
- const SEED = "promptforge";
3
- const num_days = 120;
4
- const num_users = 10_000;
5
- const avg_events_per_user_per_day = 0.83;
6
- let token = "your-mixpanel-token";
7
-
8
- // ── env overrides ──
9
- if (process.env.MP_TOKEN) token = process.env.MP_TOKEN;
10
-
1
+ // ── IMPORTS ──
11
2
  import dayjs from "dayjs";
12
3
  import utc from "dayjs/plugin/utc.js";
4
+ dayjs.extend(utc);
13
5
  import "dotenv/config";
14
6
  import * as u from "../../lib/utils/utils.js";
15
- import * as v from "ak-tools";
16
-
17
- dayjs.extend(utc);
18
- const chance = u.initChance(SEED);
19
- /** @typedef {import("../../types").Dungeon} Config */
7
+ /** @typedef {import("../../types").Dungeon} Config */
8
+
9
+ // ── OVERVIEW ──
10
+ /*
11
+ * NAME: PromptForge
12
+ * APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
13
+ * requests for chat completions, embeddings, evaluations, and tool
14
+ * use. Billing is per input/output token. Features: prompt caching,
15
+ * tool use, multi-turn conversations, batch API, model selection,
16
+ * eval pipelines.
17
+ * SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
18
+ * CORE LOOP: organization created → api key created → api call → iterate
19
+ *
20
+ * EVENTS (18):
21
+ * api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
22
+ * > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
23
+ * > model selected (3) > api key created (2) > batch job submitted (2)
24
+ * > batch job completed (2) > billing payment (2) > member invited (2)
25
+ * > organization created (1) > api key rotated (1) > webhook configured (1)
26
+ * > account deactivated (1)
27
+ *
28
+ * FUNNELS (3):
29
+ * - Onboarding: organization created → api key created → api call (70%)
30
+ * - API to Eval Pipeline: api call → tool use call → eval job (45%)
31
+ * - Usage to Billing: api call → billing payment (30%)
32
+ *
33
+ * USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
34
+ * SUPER PROPS: api_tier, primary_use_case, sdk_language
35
+ * SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
36
+ * GROUPS: none
37
+ */
20
38
 
21
- /**
22
- * ===============================================================
23
- * DATASET OVERVIEW
24
- * ===============================================================
25
- *
26
- * PromptForge -- an LLM API platform (like Anthropic/OpenAI).
27
- * Customers (developers and companies) send API requests for chat
28
- * completions, embeddings, evaluations, and tool use. Billing is
29
- * per input/output token. Key features: prompt caching, tool use,
30
- * multi-turn conversations, batch API, model selection, and
31
- * evaluation pipelines.
32
- *
33
- * - 8,000 users over 120 days, ~800K events
34
- * - Three API tiers: Free, Build, Enterprise
35
- * - Core loop: org created -> api key created -> api call -> iterate
36
- * - Revenue: token-based billing with tier-based pricing
37
- *
38
- * Key entities:
39
- * - model: LLM model version (sonnet-4, haiku-4, opus-4-6, opus-4-7)
40
- * - api_tier: Free / Build / Enterprise (determines context window, rate limits)
41
- * - tokens_used: total tokens consumed per API call (input + output)
42
- * - cost_usd: dollar cost of a single API call
43
- * - cache_enabled: prompt caching flag that reduces cost 70%
44
- * - multi_turn: whether the call is part of a conversation
45
- *
46
- * ===============================================================
47
- * ANALYTICS HOOKS (10 hooks)
48
- * ===============================================================
49
- *
50
- * NOTE: Cohort effects are HIDDEN — no flag stamping. Discoverable
51
- * only via behavioral cohorts (count event per user) or raw-prop
52
- * breakdowns (api_tier from profile, model, error_type).
53
- *
39
+ // ── HOOK STORIES ──
40
+ /*
54
41
  * ---------------------------------------------------------------
55
42
  * 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
56
43
  * ---------------------------------------------------------------
@@ -315,36 +302,273 @@ const chance = u.initChance(SEED);
315
302
  * Docs Magic Number | over billing/user | 1x | 0.7x | -30%
316
303
  */
317
304
 
305
+ // ── SCALE ──
306
+ const SEED = "promptforge";
307
+ const NUM_USERS = 10_000;
308
+ const DATASET_START = "2026-01-01T00:00:00Z";
309
+ const DATASET_END = "2026-05-01T23:59:59Z";
310
+ const EVENTS_PER_DAY = 0.83;
311
+ const token = process.env.MP_TOKEN || "your-mixpanel-token";
312
+
313
+ const chance = u.initChance(SEED);
314
+
315
+ // ── KNOBS (tweak these to reshape stories) ──
316
+ const OUTAGE_START_DAY = 40;
317
+ const OUTAGE_END_DAY = 42;
318
+ const OUTAGE_ERROR_LIKELIHOOD = 40;
319
+ const OUTAGE_LATENCY_MULT = 3;
320
+
321
+ const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
322
+ const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
323
+
324
+ const CACHE_USER_HASH_MOD = 4;
325
+ const CACHE_COST_FACTOR = 0.3;
326
+ const CACHE_ACTIVATION_PCT = 0.2;
327
+
328
+ const MODEL_MIGRATION_DAY = 60;
329
+ const MODEL_MIGRATION_LIKELIHOOD = 35;
330
+ const MODEL_MIGRATION_TOKEN_MULT = 1.5;
331
+
332
+ const AGENTIC_TOOL_THRESHOLD = 3;
333
+ const AGENTIC_MULTITURN_THRESHOLD = 3;
334
+ const AGENTIC_TOKEN_MULT = 8;
335
+ const AGENTIC_CLONE_MULT = 2;
336
+
337
+ const RATE_LIMIT_THRESHOLD = 2;
338
+ const RATE_LIMIT_KEEP_LIKELIHOOD = 40;
339
+
340
+ const BATCH_COST_FACTOR = 0.5;
341
+ const BATCH_TOKEN_MULT = 2;
342
+
343
+ const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
344
+ const EVAL_CUTOFF_DAYS = 30;
345
+
346
+ const DOCS_SWEET_MIN = 2;
347
+ const DOCS_SWEET_MAX = 4;
348
+ const DOCS_OVER_THRESHOLD = 5;
349
+ const DOCS_BILLING_BOOST = 1.35;
350
+ const DOCS_BILLING_DROP_LIKELIHOOD = 30;
351
+
352
+ const FUNNEL_TTC_ENTERPRISE = 0.67;
353
+ const FUNNEL_TTC_FREE = 1.4;
354
+
355
+ // ── HELPER FUNCTIONS ──
356
+ function handleFunnelPostHooks(record, meta) {
357
+ // H9: API-to-Eval TTC scaled by tier
358
+ const segment = meta?.profile?.api_tier;
359
+ if (Array.isArray(record) && record.length > 1) {
360
+ const factor = (
361
+ segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
362
+ segment === "Free" ? FUNNEL_TTC_FREE :
363
+ 1.0
364
+ );
365
+ if (factor !== 1.0) {
366
+ for (let i = 1; i < record.length; i++) {
367
+ const prev = dayjs(record[i - 1].time);
368
+ const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
369
+ record[i].time = prev.add(newGap, "milliseconds").toISOString();
370
+ }
371
+ }
372
+ }
373
+ return record;
374
+ }
375
+
376
+ function handleEverythingHooks(record, meta) {
377
+ const datasetStart = dayjs.unix(meta.datasetStart);
378
+ let events = record;
379
+ if (!events.length) return record;
380
+ const profile = meta && meta.profile ? meta.profile : {};
381
+
382
+ events.forEach(e => {
383
+ if (profile.api_tier) e.api_tier = profile.api_tier;
384
+ if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
385
+ if (profile.sdk_language) e.sdk_language = profile.sdk_language;
386
+ });
387
+
388
+ // H6: Outage day (days 40-41) — 40% of api calls flipped to errors
389
+ events.forEach(e => {
390
+ if (e.event !== "api call") return;
391
+ const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
392
+ if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
393
+ if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
394
+ e.is_error = true;
395
+ e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
396
+ e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
397
+ }
398
+ }
399
+ });
400
+
401
+ const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
402
+ const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
403
+
404
+ // H5: Tier-based context window & input tokens
405
+ const tier = profile.api_tier || "Free";
406
+ const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
407
+ const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
408
+ events.forEach(e => {
409
+ if (e.event === "api call") {
410
+ e.context_window = contextWindow;
411
+ e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
412
+ }
413
+ });
414
+
415
+ // H1: Prompt caching adoption — ~25% of users; activates 20% into stream
416
+ const userId = events[0] && events[0].user_id;
417
+ const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
418
+ const isCacheUser = (idHash % CACHE_USER_HASH_MOD) === 0;
419
+ if (isCacheUser) {
420
+ let cacheActivated = false;
421
+ const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
422
+ events.forEach((e, idx) => {
423
+ if (e.event === "api call") {
424
+ if (idx >= activationPoint) cacheActivated = true;
425
+ if (cacheActivated) {
426
+ e.cache_enabled = true;
427
+ e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
428
+ }
429
+ }
430
+ });
431
+ }
432
+
433
+ // H2: Model migration wave — post-day-60 Build/Enterprise calls migrate to opus-4-7
434
+ const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
435
+ if (tier === "Build" || tier === "Enterprise") {
436
+ events.forEach(e => {
437
+ if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
438
+ if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
439
+ e.model = "opus-4-7";
440
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
441
+ }
442
+ }
443
+ });
444
+ }
445
+
446
+ // H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens, 3x events
447
+ const toolUseCount = events.filter(e => e.event === "tool use call").length;
448
+ const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
449
+ const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
450
+ if (isAgenticUser) {
451
+ events.forEach(e => {
452
+ if (e.event === "api call") {
453
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
454
+ }
455
+ });
456
+ const apiCalls = events.filter(e => e.event === "api call");
457
+ const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
458
+ for (let i = 0; i < extraCount; i++) {
459
+ const template = apiCalls[i % apiCalls.length];
460
+ if (template) {
461
+ events.push({
462
+ ...template,
463
+ time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
464
+ user_id: template.user_id,
465
+ multi_turn: true,
466
+ });
467
+ }
468
+ }
469
+ }
470
+
471
+ // H4: Rate-limit churn — 2+ early rate-limit errors → drop 60% of post-week-1 events
472
+ const firstWeekEnd = firstEventTime.add(7, "days");
473
+ const earlyRateLimits = events.filter(e =>
474
+ e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
475
+ ).length;
476
+ if (earlyRateLimits >= RATE_LIMIT_THRESHOLD) {
477
+ events = events.filter(e => {
478
+ if (dayjs(e.time).isAfter(firstWeekEnd)) {
479
+ return chance.bool({ likelihood: RATE_LIMIT_KEEP_LIKELIHOOD });
480
+ }
481
+ return true;
482
+ });
483
+ }
484
+
485
+ // H7: Batch API discount — any batch job submitted → 50% cost, 2x tokens
486
+ const isBatchUser = events.some(e => e.event === "batch job submitted");
487
+ if (isBatchUser) {
488
+ events.forEach(e => {
489
+ if (e.event === "api call") {
490
+ e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
491
+ e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
492
+ }
493
+ });
494
+ }
495
+
496
+ // H8: Eval-driven retention — non-eval users lose 75% post-day-30
497
+ const hasEarlyEval = events.some(e =>
498
+ e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
499
+ );
500
+ if (!hasEarlyEval) {
501
+ const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
502
+ events = events.filter(e => {
503
+ if (dayjs(e.time).isAfter(cutoff)) {
504
+ return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
505
+ }
506
+ return true;
507
+ });
508
+ }
509
+
510
+ // H10: Docs-searched magic number — count between org-created and first billing
511
+ const orgEvent = events.find(e => e.event === "organization created");
512
+ const firstBilling = events.find(e => e.event === "billing payment");
513
+ if (orgEvent && firstBilling) {
514
+ const aTime = dayjs(orgEvent.time);
515
+ const bTime = dayjs(firstBilling.time);
516
+ const docsBetween = events.filter(e =>
517
+ e.event === "docs searched" &&
518
+ dayjs(e.time).isAfter(aTime) &&
519
+ dayjs(e.time).isBefore(bTime)
520
+ ).length;
521
+ if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
522
+ events.forEach(e => {
523
+ if (e.event === "billing payment" && typeof e.amount_usd === "number") {
524
+ e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
525
+ }
526
+ });
527
+ } else if (docsBetween >= DOCS_OVER_THRESHOLD) {
528
+ for (let i = events.length - 1; i >= 0; i--) {
529
+ if (events[i].event === "billing payment" && chance.bool({ likelihood: DOCS_BILLING_DROP_LIKELIHOOD })) {
530
+ events.splice(i, 1);
531
+ }
532
+ }
533
+ }
534
+ }
535
+
536
+ return events;
537
+ }
538
+
539
+ // ── CONFIG ──
318
540
  /** @type {Config} */
319
541
  const config = {
320
542
  version: 2,
321
- token,
322
543
  seed: SEED,
323
- datasetStart: "2026-01-01T00:00:00Z",
324
- datasetEnd: "2026-05-01T23:59:59Z",
325
- // numDays: num_days,
326
- avgEventsPerUserPerDay: avg_events_per_user_per_day,
327
- numUsers: num_users,
328
- hasAnonIds: true,
329
- avgDevicePerUser: 2,
330
- hasSessionIds: true,
544
+ datasetStart: DATASET_START,
545
+ datasetEnd: DATASET_END,
546
+ avgEventsPerUserPerDay: EVENTS_PER_DAY,
547
+ numUsers: NUM_USERS,
331
548
  format: "json",
332
549
  gzip: true,
333
- alsoInferFunnels: false,
334
- hasLocation: true,
335
- hasAndroidDevices: false,
336
- hasIOSDevices: false,
337
- hasDesktopDevices: true,
338
- hasBrowser: true,
339
- hasCampaigns: false,
340
- isAnonymous: false,
341
- hasAdSpend: false,
342
- hasAvatar: true,
550
+ credentials: {
551
+ token,
552
+ },
553
+ switches: {
554
+ hasSessionIds: true,
555
+ alsoInferFunnels: false,
556
+ hasLocation: true,
557
+ hasAndroidDevices: false,
558
+ hasIOSDevices: false,
559
+ hasDesktopDevices: true,
560
+ hasBrowser: true,
561
+ hasCampaigns: false,
562
+ isAnonymous: false,
563
+ hasAdSpend: false,
564
+ hasAvatar: true,
565
+ },
566
+ identity: {
567
+ avgDevicePerUser: 2,
568
+ },
343
569
  concurrency: 1,
344
570
  writeToDisk: false,
345
-
346
571
  soup: "growth",
347
-
348
572
  scdProps: {
349
573
  monthly_api_usage: {
350
574
  values: u.weighNumRange(0, 1000000, 0.3, 50),
@@ -359,8 +583,6 @@ const config = {
359
583
  max: 6,
360
584
  },
361
585
  },
362
-
363
- // -- Events (18) ------------------------------------------
364
586
  events: [
365
587
  {
366
588
  event: "organization created",
@@ -390,6 +612,7 @@ const config = {
390
612
  {
391
613
  event: "api call",
392
614
  weight: 10,
615
+ isStrictEvent: false,
393
616
  properties: {
394
617
  model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"],
395
618
  input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
@@ -410,6 +633,7 @@ const config = {
410
633
  {
411
634
  event: "tool use call",
412
635
  weight: 4,
636
+ isStrictEvent: false,
413
637
  properties: {
414
638
  tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
415
639
  execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
@@ -441,6 +665,7 @@ const config = {
441
665
  {
442
666
  event: "eval job",
443
667
  weight: 3,
668
+ isStrictEvent: false,
444
669
  properties: {
445
670
  eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
446
671
  num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
@@ -472,6 +697,7 @@ const config = {
472
697
  {
473
698
  event: "billing payment",
474
699
  weight: 2,
700
+ isStrictEvent: false,
475
701
  properties: {
476
702
  amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
477
703
  payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
@@ -542,8 +768,6 @@ const config = {
542
768
  },
543
769
  },
544
770
  ],
545
-
546
- // -- Funnels (3) ------------------------------------------
547
771
  funnels: [
548
772
  {
549
773
  name: "Onboarding",
@@ -571,15 +795,11 @@ const config = {
571
795
  weight: 2,
572
796
  },
573
797
  ],
574
-
575
- // -- SuperProps --------------------------------------------
576
798
  superProps: {
577
799
  api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
578
800
  primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
579
801
  sdk_language: ["python", "typescript", "java", "go", "curl"],
580
802
  },
581
-
582
- // -- UserProps ---------------------------------------------
583
803
  userProps: {
584
804
  api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
585
805
  primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
@@ -588,262 +808,9 @@ const config = {
588
808
  total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
589
809
  preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
590
810
  },
591
-
592
- // -- Hook Function ----------------------------------------
593
- hook: function (record, type, meta) {
594
-
595
- // ─────────────────────────────────────────────────────────
596
- // Hook 9 (T2C): API-TO-EVAL TIME-TO-CONVERT (funnel-post)
597
- // Enterprise users complete API to Eval Pipeline funnel 1.5x faster
598
- // (factor 0.67 on inter-event gaps); Free users 1.4x slower (factor
599
- // 1.4). Mutates record[i].time. No flag.
600
- // ─────────────────────────────────────────────────────────
601
- if (type === "funnel-post") {
602
- const segment = meta?.profile?.api_tier;
603
- if (Array.isArray(record) && record.length > 1) {
604
- const factor = (
605
- segment === "Enterprise" ? 0.67 :
606
- segment === "Free" ? 1.4 :
607
- 1.0
608
- );
609
- if (factor !== 1.0) {
610
- for (let i = 1; i < record.length; i++) {
611
- const prev = dayjs(record[i - 1].time);
612
- const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
613
- record[i].time = prev.add(newGap, "milliseconds").toISOString();
614
- }
615
- }
616
- }
617
- }
618
-
619
- // ─────────────────────────────────────────────────────────
620
- // EVERYTHING HOOKS
621
- // ─────────────────────────────────────────────────────────
622
- if (type === "everything") {
623
- const datasetStart = dayjs.unix(meta.datasetStart);
624
- let events = record;
625
- if (!events.length) return record;
626
- const profile = meta && meta.profile ? meta.profile : {};
627
-
628
- // Stamp superProps from profile for consistency
629
- events.forEach(e => {
630
- if (profile.api_tier) e.api_tier = profile.api_tier;
631
- if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
632
- if (profile.sdk_language) e.sdk_language = profile.sdk_language;
633
- });
634
-
635
- // ─────────────────────────────────────────────────────
636
- // Hook #6: OUTAGE DAY (days 40-41)
637
- // 40% of api calls get is_error=true with service errors
638
- // ─────────────────────────────────────────────────────
639
- events.forEach(e => {
640
- if (e.event !== "api call") return;
641
- const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
642
- if (dayInDataset >= 40 && dayInDataset < 42) {
643
- if (chance.bool({ likelihood: 40 })) {
644
- e.is_error = true;
645
- e.error_type = chance.pickone([
646
- "service_overloaded",
647
- "internal_server_error",
648
- "gateway_timeout",
649
- ]);
650
- e.latency_ms = Math.floor((e.latency_ms || 1500) * 3);
651
- }
652
- }
653
- });
654
-
655
- // Determine first event time for relative day calculations
656
- const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
657
- const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
658
-
659
- // ─────────────────────────────────────────────────────
660
- // Hook #5: TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER)
661
- // Scale context_window and input_tokens by tier
662
- // ─────────────────────────────────────────────────────
663
- const tier = profile.api_tier || "Free";
664
- const contextWindow = tier === "Enterprise" ? 2000000 : tier === "Build" ? 1000000 : 200000;
665
- const inputMultiplier = tier === "Enterprise" ? 4 : tier === "Build" ? 2 : 1;
666
-
667
- events.forEach(e => {
668
- if (e.event === "api call") {
669
- e.context_window = contextWindow;
670
- e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
671
- }
672
- });
673
-
674
- // ─────────────────────────────────────────────────────
675
- // Hook #1: PROMPT CACHING ADOPTION (CONVERSION)
676
- // Users with any cache_enabled=true get 70% cost reduction
677
- // on all subsequent api calls
678
- // ─────────────────────────────────────────────────────
679
- // ~25% of users have caching enabled on at least one event
680
- const userId = events[0] && events[0].user_id;
681
- const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
682
- const isCacheUser = (idHash % 4) === 0;
683
-
684
- if (isCacheUser) {
685
- let cacheActivated = false;
686
- // Activate caching on events after the first 20% of user events
687
- const activationPoint = Math.floor(events.length * 0.2);
688
- events.forEach((e, idx) => {
689
- if (e.event === "api call") {
690
- if (idx >= activationPoint) {
691
- cacheActivated = true;
692
- }
693
- if (cacheActivated) {
694
- e.cache_enabled = true;
695
- e.cost_usd = Math.round((e.cost_usd || 0.01) * 0.3 * 10000) / 10000;
696
- }
697
- }
698
- });
699
- }
700
-
701
- // ─────────────────────────────────────────────────────
702
- // Hook #2: MODEL MIGRATION WAVE
703
- // After day 60, 35% of Build/Enterprise api_calls switch to
704
- // opus-4-7 model and get 1.5x tokens_used. Reads profile.api_tier
705
- // (authoritative) and uses post-shift event timestamps.
706
- // ─────────────────────────────────────────────────────
707
- const datasetStartDay60 = datasetStart.add(60, "days");
708
- if (tier === "Build" || tier === "Enterprise") {
709
- events.forEach(e => {
710
- if (e.event === "api call" && dayjs(e.time).isAfter(datasetStartDay60)) {
711
- if (chance.bool({ likelihood: 35 })) {
712
- e.model = "opus-4-7";
713
- e.tokens_used = Math.floor((e.tokens_used || 2500) * 1.5);
714
- }
715
- }
716
- });
717
- }
718
-
719
- // ─────────────────────────────────────────────────────
720
- // Hook #3: AGENTIC LOOP POWER USERS (BEHAVIORS TOGETHER)
721
- // Users with 3+ tool use calls + 3+ multi_turn api calls
722
- // get 8x tokens, 3x events. Threshold ensures a meaningful
723
- // agentic cohort (~20-30% of users).
724
- // ─────────────────────────────────────────────────────
725
- const toolUseCount = events.filter(e => e.event === "tool use call").length;
726
- const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
727
- const isAgenticUser = toolUseCount >= 3 && multiTurnCount >= 3;
728
-
729
- if (isAgenticUser) {
730
- events.forEach(e => {
731
- if (e.event === "api call") {
732
- e.tokens_used = Math.floor((e.tokens_used || 2500) * 8);
733
- }
734
- });
735
-
736
- const apiCalls = events.filter(e => e.event === "api call");
737
- const extraCount = apiCalls.length * 2;
738
- for (let i = 0; i < extraCount; i++) {
739
- const template = apiCalls[i % apiCalls.length];
740
- if (template) {
741
- events.push({
742
- ...template,
743
- time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
744
- user_id: template.user_id,
745
- multi_turn: true,
746
- });
747
- }
748
- }
749
- }
750
-
751
- // ─────────────────────────────────────────────────────
752
- // Hook #4: RATE LIMIT CHURN
753
- // >=2 rate limit errors in first 7 days -> remove 60% of
754
- // events after week 1. Threshold is intentionally low because
755
- // avgEventsPerUserPerDay=0.83 means most users only generate
756
- // a handful of events per week.
757
- // ─────────────────────────────────────────────────────
758
- const firstWeekEnd = firstEventTime.add(7, "days");
759
- const earlyRateLimits = events.filter(e =>
760
- e.event === "rate limit error" &&
761
- dayjs(e.time).isBefore(firstWeekEnd)
762
- ).length;
763
-
764
- if (earlyRateLimits >= 2) {
765
- // Remove 60% of events after week 1
766
- events = events.filter(e => {
767
- if (dayjs(e.time).isAfter(firstWeekEnd)) {
768
- return chance.bool({ likelihood: 40 });
769
- }
770
- return true;
771
- });
772
- }
773
-
774
- // ─────────────────────────────────────────────────────
775
- // Hook #7: BATCH API DISCOUNT (PURCHASE VALUE)
776
- // Batch users get 50% lower cost_per_token, 2x tokens_used
777
- // ─────────────────────────────────────────────────────
778
- const isBatchUser = events.some(e => e.event === "batch job submitted");
779
-
780
- if (isBatchUser) {
781
- events.forEach(e => {
782
- if (e.event === "api call") {
783
- e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * 0.5 * 10000000) / 10000000;
784
- e.tokens_used = Math.floor((e.tokens_used || 2500) * 2);
785
- }
786
- });
787
- }
788
-
789
- // ─────────────────────────────────────────────────────
790
- // Hook #8: EVAL-DRIVEN RETENTION
791
- // Early eval users (first 7 days) get 75% D30 retention
792
- // Non-eval users get only 25% D30 retention (remove events)
793
- // ─────────────────────────────────────────────────────
794
- const hasEarlyEval = events.some(e =>
795
- e.event === "eval job" &&
796
- dayjs(e.time).isBefore(firstWeekEnd)
797
- );
798
-
799
- if (hasEarlyEval) {
800
- // Early eval users keep all their events (high retention)
801
- } else {
802
- // Non-eval users: remove 75% of events after day 30
803
- const day30 = firstEventTime.add(30, "days");
804
- events = events.filter(e => {
805
- if (dayjs(e.time).isAfter(day30)) {
806
- return chance.bool({ likelihood: 25 });
807
- }
808
- return true;
809
- });
810
- }
811
-
812
- // ─────────────────────────────────────────────────────
813
- // Hook 10: DOCS-SEARCHED MAGIC NUMBER (in-funnel, no flags)
814
- // Count "docs searched" events between first "organization
815
- // created" (sign-up) and any "billing payment". Sweet 2-4 → +35%
816
- // on amount_usd of billing-payment events. Over 5+ → drop 30%
817
- // of billing-payment events.
818
- // ─────────────────────────────────────────────────────
819
- const orgEvent = events.find(e => e.event === "organization created");
820
- const firstBilling = events.find(e => e.event === "billing payment");
821
- if (orgEvent && firstBilling) {
822
- const aTime = dayjs(orgEvent.time);
823
- const bTime = dayjs(firstBilling.time);
824
- const docsBetween = events.filter(e =>
825
- e.event === "docs searched" &&
826
- dayjs(e.time).isAfter(aTime) &&
827
- dayjs(e.time).isBefore(bTime)
828
- ).length;
829
- if (docsBetween >= 2 && docsBetween <= 4) {
830
- events.forEach(e => {
831
- if (e.event === "billing payment" && typeof e.amount_usd === "number") {
832
- e.amount_usd = Math.round(e.amount_usd * 1.35);
833
- }
834
- });
835
- } else if (docsBetween >= 5) {
836
- for (let i = events.length - 1; i >= 0; i--) {
837
- if (events[i].event === "billing payment" && chance.bool({ likelihood: 30 })) {
838
- events.splice(i, 1);
839
- }
840
- }
841
- }
842
- }
843
-
844
- return events;
845
- }
846
-
811
+ hook(record, type, meta) {
812
+ if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
813
+ if (type === "everything") return handleEverythingHooks(record, meta);
847
814
  return record;
848
815
  },
849
816
  };