@ak--47/dungeon-master 1.4.4 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +158 -0
  2. package/.claude/skills/create-dungeon/SKILL.md +464 -0
  3. package/.claude/skills/verify-dungeon/SKILL.md +157 -0
  4. package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
  5. package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
  6. package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
  7. package/.claude/skills/write-hooks/SKILL.md +468 -0
  8. package/CHANGELOG.md +147 -0
  9. package/HOOKS.md +1243 -597
  10. package/README.md +140 -5
  11. package/dungeons/technical/ad-spend.js +1 -1
  12. package/dungeons/technical/anonymous-users.js +1 -1
  13. package/dungeons/technical/array-of-object-lookup.js +1 -1
  14. package/dungeons/technical/datagen-v15-verify.js +74 -0
  15. package/dungeons/technical/experiments.js +1 -1
  16. package/dungeons/technical/foobar.js +1 -1
  17. package/dungeons/technical/group-analytics.js +1 -1
  18. package/dungeons/technical/mirror-strategies.js +1 -1
  19. package/dungeons/technical/nested-objects.js +1 -1
  20. package/dungeons/technical/retention-cadence.js +1 -1
  21. package/dungeons/technical/sanity.js +1 -1
  22. package/dungeons/technical/scale-test.js +1 -1
  23. package/dungeons/technical/scd.js +1 -1
  24. package/dungeons/technical/simple.js +1 -1
  25. package/dungeons/technical/simplest.js +74 -20
  26. package/dungeons/technical/text-generation.js +1 -1
  27. package/dungeons/vertical/ai-platform.js +4 -0
  28. package/dungeons/vertical/community.js +9 -3
  29. package/dungeons/vertical/crypto.js +5 -0
  30. package/dungeons/vertical/dating.js +23 -10
  31. package/dungeons/vertical/devtools.js +10 -0
  32. package/dungeons/vertical/ecommerce.js +6 -0
  33. package/dungeons/vertical/education.js +11 -0
  34. package/dungeons/vertical/fintech.js +13 -0
  35. package/dungeons/vertical/fitness.js +10 -0
  36. package/dungeons/vertical/food-delivery.js +9 -0
  37. package/dungeons/vertical/gaming.js +10 -0
  38. package/dungeons/vertical/healthcare.js +5 -0
  39. package/dungeons/vertical/insurance-application.js +10 -0
  40. package/dungeons/vertical/logistics.js +8 -1
  41. package/dungeons/vertical/marketplace.js +7 -0
  42. package/dungeons/vertical/media.js +8 -0
  43. package/dungeons/vertical/real-estate.js +7 -1
  44. package/dungeons/vertical/sass.js +12 -0
  45. package/dungeons/vertical/social.js +9 -0
  46. package/dungeons/vertical/travel.js +5 -0
  47. package/index.js +45 -7
  48. package/lib/core/config-validator.js +270 -7
  49. package/lib/core/context.js +58 -0
  50. package/lib/core/dungeon-loader.js +2 -5
  51. package/lib/generators/events.js +12 -13
  52. package/lib/generators/funnels.js +72 -1
  53. package/lib/hook-helpers/index.js +1 -0
  54. package/lib/hook-helpers/inject.js +95 -0
  55. package/lib/orchestrators/mixpanel-sender.js +27 -1
  56. package/lib/orchestrators/user-loop.js +488 -29
  57. package/lib/templates/macro-presets.js +39 -9
  58. package/lib/utils/utils.js +16 -79
  59. package/lib/verify/counting.js +320 -0
  60. package/lib/verify/emulate-breakdown.js +512 -108
  61. package/lib/verify/funnel-engine.js +539 -0
  62. package/lib/verify/identity.js +78 -0
  63. package/lib/verify/index.js +19 -0
  64. package/lib/verify/verify-dungeon.js +58 -0
  65. package/package.json +4 -2
  66. package/types.d.ts +314 -4
  67. package/scripts/smoke-test-all.mjs +0 -162
@@ -42,6 +42,7 @@ export async function userLoop(context) {
42
42
  scdProps,
43
43
  numDays,
44
44
  avgEventsPerUserPerDay,
45
+ avgActiveDaysPerUser,
45
46
  percentUsersBornInDataset = 15,
46
47
  strictEventCount = false,
47
48
  bornRecentBias = 0, // -1..1; positive = births skew toward end of window
@@ -66,7 +67,10 @@ export async function userLoop(context) {
66
67
  // Track if we've already logged the strict event count message
67
68
  let hasLoggedStrictCountReached = false;
68
69
 
69
- // Handle graceful shutdown on SIGINT (Ctrl+C)
70
+ // Handle graceful shutdown on SIGINT (Ctrl+C).
71
+ // CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
72
+ // cancellation paths leaked listeners across test runs, accumulating until
73
+ // Node fired the MaxListenersExceededWarning AND test workers stalled.
70
74
  let cancelled = false;
71
75
  const onSigint = () => {
72
76
  cancelled = true;
@@ -75,6 +79,7 @@ export async function userLoop(context) {
75
79
  };
76
80
  process.on('SIGINT', onSigint);
77
81
 
82
+ try {
78
83
  for (let i = 0; i < numUsers; i++) {
79
84
  const userPromise = USER_CONN(async () => {
80
85
  // Bail out if cancelled
@@ -104,6 +109,16 @@ export async function userLoop(context) {
104
109
  ]);
105
110
  }
106
111
 
112
+ context.reportProgress({
113
+ phase: "generation",
114
+ users: context.getUserCount(),
115
+ events: context.getEventCount(),
116
+ eps,
117
+ memory: memUsed,
118
+ elapsed: duration,
119
+ percentComplete: Math.min(100, Math.round((context.getUserCount() / numUsers) * 100))
120
+ });
121
+
107
122
  const userId = chance.guid();
108
123
  const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
109
124
  const { distinct_id, created } = user;
@@ -271,6 +286,42 @@ export async function userLoop(context) {
271
286
 
272
287
  let userFirstEventTime;
273
288
 
289
+ // ── v1.5 Active-day scheduling ──
290
+ // When `avgActiveDaysPerUser` is set, build a per-user day plan: a list
291
+ // of UTC day-start unix-seconds, one entry per planned event. Each event
292
+ // generation call pops the next day from the plan and constrains TimeSoup
293
+ // to that day's [start, end] range. Funnel events anchor on the picked
294
+ // day; subsequent funnel steps spill within `timeToConvert` hours.
295
+ //
296
+ // When unset, dayPlan stays null and behavior is fully legacy (TimeSoup
297
+ // across [adjustedCreated, FIXED_NOW]).
298
+ const soupCfgForActiveDay = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
299
+ // v1.5 follow-up (Fix #1): buildActiveDayPlan returns
300
+ // `{ plan, pickedDayBuckets }`. `pickedDayBuckets` flows into
301
+ // `applyEngagementDecay` so the decay filter never drops the last event
302
+ // on a picked day — preserving the configured distinct-day count.
303
+ const dayPlanResult = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser))
304
+ ? buildActiveDayPlan({
305
+ adjustedCreated, fixedBegin: context.FIXED_BEGIN, fixedNow: context.FIXED_NOW,
306
+ avgActiveDaysPerUser, userActiveDays,
307
+ numEvents: numEventsThisUserWillPreform,
308
+ dowWeights: soupCfgForActiveDay.dayOfWeekWeights,
309
+ chance,
310
+ })
311
+ : null;
312
+ const dayPlan = dayPlanResult ? dayPlanResult.plan : null;
313
+ const pickedDayBuckets = dayPlanResult ? dayPlanResult.pickedDayBuckets : null;
314
+ let dayPlanCursor = 0;
315
+ const nextDayBounds = () => {
316
+ if (!dayPlan || !dayPlan.length) return null;
317
+ // When the plan is exhausted, wrap (over-generation due to noise rounding;
318
+ // bounded re-use keeps remaining events on picked days rather than spilling).
319
+ const dayStartSec = dayPlan[dayPlanCursor % dayPlan.length];
320
+ dayPlanCursor++;
321
+ const dayEndSec = Math.min(dayStartSec + 86400 - 1, context.FIXED_NOW);
322
+ return { earliest: Math.max(dayStartSec, context.FIXED_BEGIN), latest: dayEndSec };
323
+ };
324
+
274
325
  const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
275
326
  .filter((f) => !f.conditions || matchConditions(profile, f.conditions))
276
327
  .reduce(weighFunnels, []);
@@ -310,7 +361,16 @@ export async function userLoop(context) {
310
361
  // PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
311
362
  if (firstFunnels.length && userIsBornInDataset) {
312
363
  const firstFunnel = chance.pickone(firstFunnels, user);
313
- let cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
364
+ // Active-day mode: anchor the first funnel on a picked day so the user's
365
+ // signup lands within their planned active window. Legacy mode: anchor at
366
+ // adjustedCreated minus a noise offset.
367
+ let cursor;
368
+ if (dayPlan) {
369
+ const bounds = nextDayBounds();
370
+ cursor = bounds ? bounds.earliest : adjustedCreated.subtract(noise(), 'seconds').unix();
371
+ } else {
372
+ cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
373
+ }
314
374
 
315
375
  // Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
316
376
  // passes = failedPriors + 1. Validator coerced bounds; default both 0.
@@ -381,20 +441,99 @@ export async function userLoop(context) {
381
441
  // Usage funnels for converted users: identity already stitched, just default 'both'.
382
442
  const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
383
443
 
384
- let usageFunnelCursor = userFirstEventTime;
385
- while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
444
+ // v1.5 follow-up (engine bunchiness fix, 2026-05-09):
445
+ // REMOVED `usageFunnelCursor` accumulator. Each funnel call now uses
446
+ // `userFirstEventTime` (constant) as the anchor. Without this, the cursor
447
+ // chained `last_event_time + small_gap` between funnel runs, walking past
448
+ // FIXED_NOW and producing the right-edge bunchiness regression. See
449
+ // `plans/ENGINE-BUNCHINESS/FIX.md` for the full diagnosis.
450
+ //
451
+ // Loop budget now counts SURVIVING events (post `_drop`), not raw output.
452
+ // Combined with the cursor removal, this gives each funnel an independent
453
+ // uniform-in-window anchor (via TimeSoup) and lets the budget loop iterate
454
+ // until the user actually has the target event count.
455
+ //
456
+ // v1.5: when auto-promote marks every event strict (e.g. all events appear in
457
+ // the only funnel), there's nothing for the standalone branch to pick. Bail
458
+ // out cleanly rather than crashing on `pick([])`.
459
+ const hasUsageFunnels = usageFunnels.length > 0;
460
+ const hasStandaloneEvents = weightedEvents.length > 0;
461
+ // Hard ceiling on iterations: defends against pathological configs where
462
+ // every funnel produces 0 surviving events (e.g. ttc > numDays so step1
463
+ // uniform anchor always lands too late). Cap at 2× expected iteration count
464
+ // based on average funnel length (~7 steps) to avoid infinite loops.
465
+ const MAX_ITERATIONS = Math.max(100, numEventsThisUserWillPreform * 2);
466
+ let iterationCount = 0;
467
+ while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled && iterationCount < MAX_ITERATIONS) {
468
+ iterationCount++;
386
469
  let newEvents;
387
- if (usageFunnels.length && userConverted) {
470
+ // v1.5 active-day: pop next picked-day bounds. Pass `latestTime` through
471
+ // featureCtx so makeEvent's TimeSoup confines to that day. Funnel cursor
472
+ // gets re-anchored to the picked day's start (subsequent funnel steps
473
+ // spill within `timeToConvert` hours; this is intentional).
474
+ const dayBounds = dayPlan ? nextDayBounds() : null;
475
+ // Compute step1's `latestTime` so the funnel's relative span fits before
476
+ // FIXED_NOW. Without this, born-late users + long-ttc funnels generate
477
+ // large numbers of `_drop`'d events that consume budget cycles. The
478
+ // safety margin is `timeToConvert * 3600 - 1` seconds (matches v1.5
479
+ // conversion window contract).
480
+ const standaloneFeatureCtx = dayBounds
481
+ ? { ...featureCtx, latestTime: dayBounds.latest }
482
+ : featureCtx;
483
+ // Nothing to generate: user converted via firstFunnel only and has no
484
+ // usage funnels and no standalone events. Stop attempting.
485
+ if ((!hasUsageFunnels || !userConverted) && !hasStandaloneEvents) break;
486
+ if (hasUsageFunnels && userConverted) {
388
487
  const currentFunnel = chance.pickone(usageFunnels);
389
- const [data, converted] = await makeFunnel(context, currentFunnel, user, usageFunnelCursor, profile, userSCD, persona, featureCtx, usageAttemptMeta);
488
+ const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
489
+ // Anchor cursor at picked day's start when active-day mode is on,
490
+ // otherwise pass userFirstEventTime (constant). NO cursor accumulation.
491
+ const funnelCursor = dayBounds ? dayBounds.earliest : userFirstEventTime;
492
+ // Constrain funnel step1's TimeSoup latestTime so the full funnel fits in
493
+ // window. Without this, late steps spill past FIXED_NOW and get `_drop`'d.
494
+ //
495
+ // v1.5 final (2026-05-09): `FUNNEL_DEAD_ZONE_CAP_SEC = 0` — funnels can
496
+ // anchor step1 right up to FIXED_NOW. Earlier rounds defended against a
497
+ // cursor-accumulation bug by reserving a `ttc`-sized dead zone at the
498
+ // right edge; round 1 fixed cursor accumulation directly, leaving the
499
+ // dead zone as defense-in-depth. The future-time guard at storage step
500
+ // 14 (per CLAUDE.md "Execution Order") drops any event with `time >
501
+ // FIXED_NOW`, so spillover from late funnel steps is filtered there
502
+ // instead of by anchoring upstream. Removing the dead zone eliminated
503
+ // the last-day cliff for funnel-heavy dungeons WITHOUT re-introducing
504
+ // `futureEvents > 0` — verified across the 194-combo engine-validation
505
+ // sweep (`scripts/sweep-engine.mjs`, `plans/ENGINE-VALIDATION/FIX.md`).
506
+ //
507
+ // Trade-off retained: long-ttc funnels still lose some late steps to
508
+ // `_drop`. Budget loop iterates more to compensate. Catch-all funnel
509
+ // (`ttc=1d`, set in config-validator) is unaffected.
510
+ //
511
+ // Born-late edge case: when `funnelCursor > FN`, the `safeLatest >
512
+ // funnelCursor` check below falls back to `FN` so the user can emit.
513
+ const FUNNEL_DEAD_ZONE_CAP_SEC = 0;
514
+ const deadZoneSec = Math.min(ttcSec, FUNNEL_DEAD_ZONE_CAP_SEC);
515
+ const safeLatest = context.FIXED_NOW - deadZoneSec;
516
+ const funnelLatestTime = dayBounds
517
+ ? dayBounds.latest
518
+ : (safeLatest > funnelCursor ? safeLatest : context.FIXED_NOW);
519
+ const funnelEventFeatureCtx = { ...featureCtx, latestTime: funnelLatestTime };
520
+ const [data, converted] = await makeFunnel(context, currentFunnel, user, funnelCursor, profile, userSCD, persona, funnelEventFeatureCtx, usageAttemptMeta);
521
+ // Budget counts raw output (matches pre-fix semantics). For short-ttc
522
+ // funnels (≤1d), almost no events `_drop` so the loop terminates at the
523
+ // expected count. For long-ttc funnels (>1d), some late steps `_drop` —
524
+ // loop iterates more to compensate, world-event `_drop`s still reduce
525
+ // the user's surviving total since they fire on dropped events too.
390
526
  numEventsPreformed += data.length;
391
527
  newEvents = data;
392
- if (data.length) {
393
- const lastTime = dayjs(data[data.length - 1].time).unix();
394
- usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
395
- }
396
528
  } else {
397
- const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, true, false, featureCtx, standaloneIdentityCtx);
529
+ // Active-day mode: standalone event uses picked-day start as earliestTime
530
+ // + day-end as latestTime (passed via eventFeatureCtx).
531
+ // `isFirstEvent: false` (8th arg) so TimeSoup distributes the timestamp
532
+ // — passing `true` here would pin every standalone event to the same
533
+ // `earliestTime`, which the now-deleted bunchIntoSessions used to paper
534
+ // over. With bunchIntoSessions removed, TimeSoup is the time source.
535
+ const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
536
+ const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
398
537
  numEventsPreformed++;
399
538
  newEvents = [data];
400
539
  }
@@ -424,7 +563,7 @@ export async function userLoop(context) {
424
563
  const userDecay = persona?.engagementDecay || globalEngagementDecay;
425
564
  if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
426
565
  // adjustedCreated and event times now share the same dataset window — no shift.
427
- usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
566
+ usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance, pickedDayBuckets);
428
567
  }
429
568
 
430
569
  // Feature 4: Data quality — duplicates and late-arriving
@@ -450,18 +589,17 @@ export async function userLoop(context) {
450
589
  }
451
590
  }
452
591
 
453
- // Session clustering: redistribute events into temporal bursts, then assign session IDs
592
+ // Session clustering: assign session IDs based on natural temporal gaps.
593
+ // v1.5: bunchIntoSessions deleted — was a redundant wholesale time-overwrite
594
+ // that scrambled multi-step funnels (round-robin into anchor buckets) and
595
+ // clobbered v1.5 active-day picking. assignSessionIds operates on the original
596
+ // TimeSoup-driven timestamps (Mixpanel-aligned 30-min-gap rule).
454
597
  if (hasSessionIds && usersEvents.length > 0) {
455
- const soupCfg = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
456
- const defaultPeaks = Math.max(5, (config.numDays || 30) * 2);
457
- const { mean: soupMean = 0, deviation: soupDev = 2, peaks: soupPeaks = defaultPeaks,
458
- dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD } = soupCfg;
459
-
460
- u.bunchIntoSessions(usersEvents, sessionTimeout, {
461
- earliestTime: userFirstEventTime,
462
- latestTime: context.FIXED_NOW,
463
- peaks: soupPeaks, deviation: soupDev, mean: soupMean,
464
- dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
598
+ // assignSessionIds requires events sorted ascending by time.
599
+ usersEvents.sort((a, b) => {
600
+ const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
601
+ const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
602
+ return ta - tb;
465
603
  });
466
604
  u.assignSessionIds(usersEvents, sessionTimeout);
467
605
 
@@ -483,6 +621,15 @@ export async function userLoop(context) {
483
621
  }
484
622
  }
485
623
 
624
+ // v1.5: Touchpoint cap. Sample up to `maxTouchpointsPerUser` (default 10,
625
+ // matching Mixpanel `TOUCHPOINTS_LIMIT`) eligible events from the user's
626
+ // lifetime and stamp UTMs on the sample. Lifetime-distributed sampling
627
+ // preserves realistic touch shape — last-10-window attribution then
628
+ // gives meaningful first/last-touch results.
629
+ if (config.hasCampaigns && usersEvents.length > 0) {
630
+ applyTouchpointCap(usersEvents, config, defaults, chance);
631
+ }
632
+
486
633
  // Hook for processing all user events (hooks override everything)
487
634
  if (config.hook) {
488
635
  // `meta.isPreAuth(event)` predicate bound to this user's auth state.
@@ -511,14 +658,38 @@ export async function userLoop(context) {
511
658
  if (Array.isArray(newEvents)) usersEvents = newEvents;
512
659
  }
513
660
 
661
+ // v1.5: auto-sort by time after everything hook. Defends against the
662
+ // most common new footgun — hooks that push() cloned events with
663
+ // arbitrary timestamps and break the greedy funnel engine's
664
+ // chronological-order requirement. Opt out with `autoSortAfterEverything: false`.
665
+ if (config.autoSortAfterEverything !== false && usersEvents.length > 1) {
666
+ usersEvents.sort((a, b) => {
667
+ const ta = (a && typeof a.time === 'string') ? Date.parse(a.time) : Number(a && a.time);
668
+ const tb = (b && typeof b.time === 'string') ? Date.parse(b.time) : Number(b && b.time);
669
+ if (!Number.isFinite(ta) && !Number.isFinite(tb)) return 0;
670
+ if (!Number.isFinite(ta)) return 1;
671
+ if (!Number.isFinite(tb)) return -1;
672
+ return ta - tb;
673
+ });
674
+ }
675
+
514
676
  // Defensive guard: drop any events whose timestamp landed past the
515
677
  // configured dataset end. Hooks that duplicate events with time offsets
516
678
  // (weekend surges, viral spreads) can leak a few past the boundary.
679
+ // v1.5 follow-up (`reccomendations-agent-1.md` Fix #2): surface the drop
680
+ // in verbose mode. Silent dropping made determinism failures hard to debug
681
+ // — events vanished without trace. Per-user log helps the owner see which
682
+ // users + how many events were affected.
683
+ const beforeFutureFilter = usersEvents.length;
517
684
  usersEvents = usersEvents.filter(e => {
518
685
  if (!e || !e.time) return true;
519
686
  const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
520
687
  return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
521
688
  });
689
+ const droppedFuture = beforeFutureFilter - usersEvents.length;
690
+ if (droppedFuture > 0 && config.verbose) {
691
+ console.warn(`⚠️ Dropped ${droppedFuture} future-dated event(s) for user ${distinct_id}`);
692
+ }
522
693
 
523
694
  // Store all user data
524
695
  await userProfilesData.hookPush(profile);
@@ -552,9 +723,11 @@ export async function userLoop(context) {
552
723
  if (dataQuality && dataQuality.botUsers > 0) {
553
724
  await generateBotUsers(context, dataQuality, storage);
554
725
  }
555
-
556
- // Clean up SIGINT handler
557
- process.removeListener('SIGINT', onSigint);
726
+ } finally {
727
+ // Always remove the SIGINT listener — even if userLoop throws or is
728
+ // cancelled. Pre-fix this leaked across test runs and stalled workers.
729
+ process.removeListener('SIGINT', onSigint);
730
+ }
558
731
  }
559
732
 
560
733
 
@@ -573,12 +746,261 @@ export function matchConditions(profile, conditions) {
573
746
  return true;
574
747
  }
575
748
 
749
+ // ── v1.5 Active-Day Plan Helpers ──
750
+
751
+ /**
752
+ * Build a deterministic per-user "day plan" for active-day mode.
753
+ *
754
+ * Each element of `plan` names the day on which the event-i should land. Events
755
+ * naturally concentrate onto `targetActiveDays` distinct days (drawn from a normal
756
+ * around `avgActiveDaysPerUser`, clamped to `[1, userActiveDays]`).
757
+ *
758
+ * Day picking uses weighted-without-replacement against soup DOW weights so the
759
+ * cohort-level weekly rhythm is preserved. Event distribution across picked days
760
+ * is proportional to those same weights, with a floor of 1 event per picked day.
761
+ *
762
+ * **Return shape (v1.5 follow-up — `reccomendations-agent-1.md` Fix #1):**
763
+ * `{ plan, pickedDayBuckets }` — `plan` is the shuffled per-event day-start
764
+ * unix-seconds array; `pickedDayBuckets` is a `Set<number>` of UTC-day-index
765
+ * buckets (`Math.floor(timestamp_ms / 86400000)`). Downstream consumers like
766
+ * `applyEngagementDecay` use `pickedDayBuckets` to enforce the v1.5
767
+ * distinct-day contract — see Fix #1 below.
768
+ *
769
+ * @param {Object} args
770
+ * @param {import('dayjs').Dayjs} args.adjustedCreated - User's "created" timestamp
771
+ * @param {number} args.fixedBegin - Dataset start (unix seconds)
772
+ * @param {number} args.fixedNow - Dataset end (unix seconds)
773
+ * @param {number} args.avgActiveDaysPerUser
774
+ * @param {number} args.userActiveDays - Capacity (max possible distinct days)
775
+ * @param {number} args.numEvents - Total events to schedule for this user
776
+ * @param {number[]} [args.dowWeights] - 7-element soup DOW weights (Sun..Sat)
777
+ * @param {Object} args.chance - Seeded chance instance
778
+ * @returns {{ plan: number[], pickedDayBuckets: Set<number> } | null} Day plan
779
+ * + bucket set, or null if not applicable (no candidate days, no events, etc.).
780
+ */
781
+ function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, chance }) {
782
+ if (!Number.isFinite(numEvents) || numEvents <= 0) return null;
783
+
784
+ // Candidate day buckets: UTC days intersecting [max(adjustedCreated, FIXED_BEGIN), FIXED_NOW].
785
+ // Pre-existing users with 'uniform' preExistingSpread have adjustedCreated < FIXED_BEGIN;
786
+ // active-day mode constrains them to in-window days only (matches what Mixpanel sees).
787
+ const userStartUnix = Math.max(
788
+ Math.floor(adjustedCreated.unix()),
789
+ Math.floor(fixedBegin)
790
+ );
791
+ const userEndUnix = Math.floor(fixedNow);
792
+ if (userEndUnix < userStartUnix) return null;
793
+
794
+ const dayMs = 86400;
795
+ const firstDay = Math.floor(userStartUnix / dayMs);
796
+ const lastDay = Math.floor(userEndUnix / dayMs);
797
+ const candidateDays = [];
798
+ for (let d = firstDay; d <= lastDay; d++) {
799
+ candidateDays.push(d * dayMs);
800
+ }
801
+ if (!candidateDays.length) return null;
802
+
803
+ // Weight by soup DOW (Sun=0..Sat=6). Default uniform if no weights configured.
804
+ const weights = candidateDays.map(daySec => {
805
+ if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
806
+ const dow = new Date(daySec * 1000).getUTCDay();
807
+ const w = Number(dowWeights[dow]);
808
+ return Number.isFinite(w) && w > 0 ? w : 0.0001;
809
+ });
810
+
811
+ // Draw target active-day count: normal(mean, sd=mean/3), clamped.
812
+ const meanActive = Math.max(1, Math.min(avgActiveDaysPerUser, candidateDays.length));
813
+ const sd = Math.max(0.5, meanActive / 3);
814
+ let targetActiveDays = Math.round(chance.normal({ mean: meanActive, dev: sd }));
815
+ targetActiveDays = Math.max(1, Math.min(targetActiveDays, candidateDays.length));
816
+
817
+ // Pick which days are active.
818
+ const pickedDays = weightedSampleNoReplacement(candidateDays, weights, targetActiveDays, chance);
819
+ if (!pickedDays.length) return null;
820
+ pickedDays.sort((a, b) => a - b);
821
+
822
+ // Recompute weights aligned to picked days for event distribution.
823
+ const pickedWeights = pickedDays.map(daySec => {
824
+ if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
825
+ const dow = new Date(daySec * 1000).getUTCDay();
826
+ const w = Number(dowWeights[dow]);
827
+ return Number.isFinite(w) && w > 0 ? w : 0.0001;
828
+ });
829
+ const totalWeight = pickedWeights.reduce((s, x) => s + x, 0);
830
+
831
+ // Allocate event counts across picked days. Each picked day gets at least 1
832
+ // event so every day is "used" — distinct-day count actually hits target.
833
+ const k = pickedDays.length;
834
+ let allocations;
835
+ if (numEvents <= k) {
836
+ // Fewer events than picked days: distribute one each, sample which days get them.
837
+ const subPicks = weightedSampleNoReplacement(pickedDays, pickedWeights, numEvents, chance);
838
+ const subSet = new Set(subPicks);
839
+ allocations = pickedDays.map(d => subSet.has(d) ? 1 : 0);
840
+ } else {
841
+ // Floor 1 per picked day, distribute remainder proportionally.
842
+ const remainder = numEvents - k;
843
+ allocations = pickedWeights.map(w => Math.floor((remainder * w) / totalWeight));
844
+ // Add floor of 1 to each
845
+ for (let i = 0; i < k; i++) allocations[i] += 1;
846
+ // Distribute rounding remainder by descending weight order
847
+ let assigned = allocations.reduce((s, x) => s + x, 0);
848
+ let rem = numEvents - assigned;
849
+ const order = pickedDays.map((_, i) => i).sort((a, b) => pickedWeights[b] - pickedWeights[a]);
850
+ for (let i = 0; i < rem; i++) {
851
+ allocations[order[i % order.length]]++;
852
+ }
853
+ }
854
+
855
+ // Expand into a flat plan and shuffle deterministically.
856
+ const plan = [];
857
+ for (let i = 0; i < k; i++) {
858
+ for (let j = 0; j < allocations[i]; j++) {
859
+ plan.push(pickedDays[i]);
860
+ }
861
+ }
862
+ // v1.5 follow-up (Fix #1): expose the picked-day bucket set so engagement
863
+ // decay can protect the last surviving event per picked day. Buckets use
864
+ // the same `floor(ms / 86400000)` formula that decay applies to event times.
865
+ const pickedDayBuckets = new Set(pickedDays.map(daySec => Math.floor(daySec / 86400)));
866
+ // chance.shuffle uses seeded RNG → deterministic.
867
+ return { plan: chance.shuffle(plan), pickedDayBuckets };
868
+ }
869
+
870
+ /**
871
+ * v1.5 Touchpoint cap: sample up to `maxTouchpointsPerUser` eligible events from
872
+ * the user's stream and stamp UTMs on the sample. Mirrors Mixpanel's
873
+ * `TOUCHPOINTS_LIMIT = 10` (`backend/libquery/properties_over_time/attributed_value_reader.cpp`).
874
+ *
875
+ * Eligibility:
876
+ * - If any event in `config.events[]` has `isAttributionEvent: true`: candidate
877
+ * pool = events whose name carries that flag.
878
+ * - Else (legacy fallback): candidate pool = ALL events in the user's stream.
879
+ *
880
+ * Sampling:
881
+ * - If `eligible.length <= cap`: stamp all of them.
882
+ * - Else: uniform random sample of size `cap` (seeded `chance.pickset`,
883
+ * deterministic). Sort sample chronologically before stamping so UTMs land
884
+ * in time order.
885
+ *
886
+ * Mutates `events` in place.
887
+ *
888
+ * @param {Object[]} events - User's events (flat shape with .event, .time)
889
+ * @param {Object} config - Validated dungeon config
890
+ * @param {Object} defaults - Context.defaults (provides campaigns())
891
+ * @param {Object} chance - Seeded chance instance
892
+ */
893
+ function applyTouchpointCap(events, config, defaults, chance) {
894
+ const cap = Number.isFinite(config.maxTouchpointsPerUser)
895
+ ? config.maxTouchpointsPerUser
896
+ : 10;
897
+ if (cap <= 0) return;
898
+
899
+ // Build event-name → config map for isAttributionEvent lookup.
900
+ const eventCfgByName = new Map();
901
+ for (const e of (config.events || [])) {
902
+ if (e && e.event) eventCfgByName.set(e.event, e);
903
+ }
904
+
905
+ // Determine eligible events.
906
+ let eligible;
907
+ if (config.hasAttributionFlags) {
908
+ eligible = events.filter(e => {
909
+ if (!e || !e.event) return false;
910
+ const cfg = eventCfgByName.get(e.event);
911
+ return cfg && cfg.isAttributionEvent === true;
912
+ });
913
+ } else {
914
+ eligible = events.slice();
915
+ }
916
+ if (eligible.length === 0) return;
917
+
918
+ // Sample up to cap (uniform random without replacement, seeded).
919
+ let sample;
920
+ if (eligible.length <= cap) {
921
+ sample = eligible;
922
+ } else {
923
+ sample = chance.pickset(eligible, cap);
924
+ }
925
+
926
+ // Sort sample chronologically so UTMs land in time order.
927
+ sample.sort((a, b) => {
928
+ const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
929
+ const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
930
+ return ta - tb;
931
+ });
932
+
933
+ // Stamp UTMs on each sampled event using a campaign template.
934
+ for (const ev of sample) {
935
+ const campaignTemplate = u.pickRandom(defaults.campaigns());
936
+ if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
937
+ for (const [k, v] of Object.entries(campaignTemplate)) {
938
+ ev[k] = u.choose(v);
939
+ }
940
+ }
941
+ }
942
+
943
+ /**
944
+ * Weighted-without-replacement sampler. Samples `count` items from `items` without
945
+ * replacement, with selection probability proportional to `weights`.
946
+ *
947
+ * @param {*[]} items
948
+ * @param {number[]} weights
949
+ * @param {number} count
950
+ * @param {Object} chance - Seeded chance instance
951
+ * @returns {*[]}
952
+ */
953
+ function weightedSampleNoReplacement(items, weights, count, chance) {
954
+ if (count >= items.length) return items.slice();
955
+ const pool = items.slice();
956
+ const w = weights.slice();
957
+ const result = [];
958
+ for (let i = 0; i < count; i++) {
959
+ const total = w.reduce((s, x) => s + x, 0);
960
+ let chosen;
961
+ if (total <= 0) {
962
+ chosen = chance.integer({ min: 0, max: pool.length - 1 });
963
+ } else {
964
+ let roll = chance.floating({ min: 0, max: total });
965
+ chosen = 0;
966
+ for (let j = 0; j < w.length; j++) {
967
+ roll -= w[j];
968
+ if (roll <= 0) { chosen = j; break; }
969
+ }
970
+ }
971
+ result.push(pool[chosen]);
972
+ pool.splice(chosen, 1);
973
+ w.splice(chosen, 1);
974
+ }
975
+ return result;
976
+ }
977
+
576
978
  // ── Advanced Feature Helper Functions ──
577
979
 
578
980
  /**
579
- * Feature 3: Apply engagement decay to a user's events
981
+ * Feature 3: Apply engagement decay to a user's events.
982
+ *
983
+ * Filters events probabilistically per the configured decay model
984
+ * (`exponential` / `linear` / `step`). Late events drop more often.
985
+ *
986
+ * **v1.5 follow-up (`reccomendations-agent-1.md` Fix #1):** when
987
+ * `pickedDayBuckets` is provided (active-day mode), the filter NEVER drops the
988
+ * last surviving event on any picked day. Without this, decay can silently
989
+ * undershoot the configured `avgActiveDaysPerUser` by killing all events on
990
+ * sparse late days (e.g., exponential decay with short half-life). The protect-
991
+ * last-event logic preserves the v1.5 distinct-day contract.
992
+ *
993
+ * @param {Object[]} events - User's events (mutated by filter; new array returned)
994
+ * @param {Object} decay - `engagementDecay` config (model/halfLife/floor/etc.)
995
+ * @param {*} userCreated - dayjs object or ISO string of user's creation time
996
+ * @param {Object} context - Engine context (unused; kept for future use)
997
+ * @param {Object} chance - Seeded chance instance
998
+ * @param {Set<number> | null} [pickedDayBuckets=null] - From `buildActiveDayPlan`.
999
+ * Each bucket = `Math.floor(timestamp_ms / 86400000)`. When non-null, the
1000
+ * filter protects the last surviving event on each bucket.
1001
+ * @returns {Object[]} Filtered events array
580
1002
  */
581
- function applyEngagementDecay(events, decay, userCreated, context, chance) {
1003
+ function applyEngagementDecay(events, decay, userCreated, context, chance, pickedDayBuckets = null) {
582
1004
  if (!events.length) return events;
583
1005
  // Perf 2: Use Date.parse instead of dayjs for hot loop
584
1006
  const userStartUnix = new Date(userCreated.toISOString ? userCreated.toISOString() : userCreated).getTime() / 1000;
@@ -587,6 +1009,23 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
587
1009
  const reactivationChance = decay.reactivationChance || 0;
588
1010
  const reactivationMult = decay.reactivationMultiplier || 2.0;
589
1011
 
1012
+ // v1.5 Fix #1: pre-compute per-bucket counts so we can enforce "at least one
1013
+ // event survives on each picked day". Only counts events that fall within a
1014
+ // picked-day bucket (events spilled to other days don't count toward the
1015
+ // per-bucket survivor budget).
1016
+ const dayMs = 86400000;
1017
+ const dayCounts = new Map(); // bucket → remaining (mutable) event count
1018
+ if (pickedDayBuckets && pickedDayBuckets.size) {
1019
+ for (const ev of events) {
1020
+ const t = new Date(ev.time).getTime();
1021
+ if (!Number.isFinite(t)) continue;
1022
+ const bucket = Math.floor(t / dayMs);
1023
+ if (pickedDayBuckets.has(bucket)) {
1024
+ dayCounts.set(bucket, (dayCounts.get(bucket) || 0) + 1);
1025
+ }
1026
+ }
1027
+ }
1028
+
590
1029
  return events.filter(ev => {
591
1030
  const evUnix = new Date(ev.time).getTime() / 1000;
592
1031
  const daysSinceBirth = Math.max(0, (evUnix - userStartUnix) / 86400);
@@ -611,7 +1050,27 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
611
1050
  retention = Math.min(1.0, retention * reactivationMult);
612
1051
  }
613
1052
 
614
- return chance.bool({ likelihood: retention * 100 });
1053
+ const keep = chance.bool({ likelihood: retention * 100 });
1054
+
1055
+ // v1.5 Fix #1: protect the last surviving event on each picked day.
1056
+ // Determinism preserved: no extra RNG calls, just a count check.
1057
+ if (pickedDayBuckets && pickedDayBuckets.size) {
1058
+ const bucket = Math.floor(new Date(ev.time).getTime() / dayMs);
1059
+ if (pickedDayBuckets.has(bucket)) {
1060
+ const remaining = dayCounts.get(bucket) || 0;
1061
+ if (!keep) {
1062
+ if (remaining <= 1) {
1063
+ // Last event on this picked day — protect it. Leave dayCounts
1064
+ // at 1 so subsequent events on this bucket can still drop.
1065
+ return true;
1066
+ }
1067
+ dayCounts.set(bucket, remaining - 1);
1068
+ }
1069
+ // If keep===true, no decrement — the survivor still counts.
1070
+ }
1071
+ }
1072
+
1073
+ return keep;
615
1074
  });
616
1075
  }
617
1076