@ak--47/dungeon-master 1.4.5 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +158 -0
  2. package/.claude/skills/create-dungeon/SKILL.md +464 -0
  3. package/.claude/skills/verify-dungeon/SKILL.md +157 -0
  4. package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
  5. package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
  6. package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
  7. package/.claude/skills/write-hooks/SKILL.md +468 -0
  8. package/CHANGELOG.md +139 -0
  9. package/HOOKS.md +1243 -597
  10. package/README.md +140 -5
  11. package/dungeons/technical/ad-spend.js +1 -1
  12. package/dungeons/technical/anonymous-users.js +1 -1
  13. package/dungeons/technical/array-of-object-lookup.js +1 -1
  14. package/dungeons/technical/datagen-v15-verify.js +74 -0
  15. package/dungeons/technical/experiments.js +1 -1
  16. package/dungeons/technical/foobar.js +1 -1
  17. package/dungeons/technical/group-analytics.js +1 -1
  18. package/dungeons/technical/mirror-strategies.js +1 -1
  19. package/dungeons/technical/nested-objects.js +1 -1
  20. package/dungeons/technical/retention-cadence.js +1 -1
  21. package/dungeons/technical/sanity.js +1 -1
  22. package/dungeons/technical/scale-test.js +1 -1
  23. package/dungeons/technical/scd.js +1 -1
  24. package/dungeons/technical/simple.js +1 -1
  25. package/dungeons/technical/simplest.js +74 -20
  26. package/dungeons/technical/text-generation.js +1 -1
  27. package/dungeons/vertical/ai-platform.js +4 -0
  28. package/dungeons/vertical/community.js +9 -3
  29. package/dungeons/vertical/crypto.js +5 -0
  30. package/dungeons/vertical/dating.js +23 -10
  31. package/dungeons/vertical/devtools.js +10 -0
  32. package/dungeons/vertical/ecommerce.js +6 -0
  33. package/dungeons/vertical/education.js +11 -0
  34. package/dungeons/vertical/fintech.js +13 -0
  35. package/dungeons/vertical/fitness.js +10 -0
  36. package/dungeons/vertical/food-delivery.js +9 -0
  37. package/dungeons/vertical/gaming.js +10 -0
  38. package/dungeons/vertical/healthcare.js +5 -0
  39. package/dungeons/vertical/insurance-application.js +10 -0
  40. package/dungeons/vertical/logistics.js +8 -1
  41. package/dungeons/vertical/marketplace.js +7 -0
  42. package/dungeons/vertical/media.js +8 -0
  43. package/dungeons/vertical/real-estate.js +7 -1
  44. package/dungeons/vertical/sass.js +12 -0
  45. package/dungeons/vertical/social.js +9 -0
  46. package/dungeons/vertical/travel.js +5 -0
  47. package/index.js +19 -4
  48. package/lib/core/config-validator.js +270 -7
  49. package/lib/core/dungeon-loader.js +2 -5
  50. package/lib/generators/events.js +12 -13
  51. package/lib/generators/funnels.js +72 -1
  52. package/lib/hook-helpers/index.js +1 -0
  53. package/lib/hook-helpers/inject.js +95 -0
  54. package/lib/orchestrators/user-loop.js +478 -29
  55. package/lib/templates/macro-presets.js +39 -9
  56. package/lib/utils/utils.js +16 -79
  57. package/lib/verify/counting.js +320 -0
  58. package/lib/verify/emulate-breakdown.js +512 -108
  59. package/lib/verify/funnel-engine.js +539 -0
  60. package/lib/verify/identity.js +78 -0
  61. package/lib/verify/index.js +19 -0
  62. package/lib/verify/verify-dungeon.js +58 -0
  63. package/package.json +4 -2
  64. package/types.d.ts +237 -4
  65. package/scripts/smoke-test-all.mjs +0 -162
@@ -42,6 +42,7 @@ export async function userLoop(context) {
42
42
  scdProps,
43
43
  numDays,
44
44
  avgEventsPerUserPerDay,
45
+ avgActiveDaysPerUser,
45
46
  percentUsersBornInDataset = 15,
46
47
  strictEventCount = false,
47
48
  bornRecentBias = 0, // -1..1; positive = births skew toward end of window
@@ -66,7 +67,10 @@ export async function userLoop(context) {
66
67
  // Track if we've already logged the strict event count message
67
68
  let hasLoggedStrictCountReached = false;
68
69
 
69
- // Handle graceful shutdown on SIGINT (Ctrl+C)
70
+ // Handle graceful shutdown on SIGINT (Ctrl+C).
71
+ // CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
72
+ // cancellation paths leaked listeners across test runs, accumulating until
73
+ // Node fired the MaxListenersExceededWarning AND test workers stalled.
70
74
  let cancelled = false;
71
75
  const onSigint = () => {
72
76
  cancelled = true;
@@ -75,6 +79,7 @@ export async function userLoop(context) {
75
79
  };
76
80
  process.on('SIGINT', onSigint);
77
81
 
82
+ try {
78
83
  for (let i = 0; i < numUsers; i++) {
79
84
  const userPromise = USER_CONN(async () => {
80
85
  // Bail out if cancelled
@@ -281,6 +286,42 @@ export async function userLoop(context) {
281
286
 
282
287
  let userFirstEventTime;
283
288
 
289
+ // ── v1.5 Active-day scheduling ──
290
+ // When `avgActiveDaysPerUser` is set, build a per-user day plan: a list
291
+ // of UTC day-start unix-seconds, one entry per planned event. Each event
292
+ // generation call pops the next day from the plan and constrains TimeSoup
293
+ // to that day's [start, end] range. Funnel events anchor on the picked
294
+ // day; subsequent funnel steps spill within `timeToConvert` hours.
295
+ //
296
+ // When unset, dayPlan stays null and behavior is fully legacy (TimeSoup
297
+ // across [adjustedCreated, FIXED_NOW]).
298
+ const soupCfgForActiveDay = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
299
+ // v1.5 follow-up (Fix #1): buildActiveDayPlan returns
300
+ // `{ plan, pickedDayBuckets }`. `pickedDayBuckets` flows into
301
+ // `applyEngagementDecay` so the decay filter never drops the last event
302
+ // on a picked day — preserving the configured distinct-day count.
303
+ const dayPlanResult = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser))
304
+ ? buildActiveDayPlan({
305
+ adjustedCreated, fixedBegin: context.FIXED_BEGIN, fixedNow: context.FIXED_NOW,
306
+ avgActiveDaysPerUser, userActiveDays,
307
+ numEvents: numEventsThisUserWillPreform,
308
+ dowWeights: soupCfgForActiveDay.dayOfWeekWeights,
309
+ chance,
310
+ })
311
+ : null;
312
+ const dayPlan = dayPlanResult ? dayPlanResult.plan : null;
313
+ const pickedDayBuckets = dayPlanResult ? dayPlanResult.pickedDayBuckets : null;
314
+ let dayPlanCursor = 0;
315
+ const nextDayBounds = () => {
316
+ if (!dayPlan || !dayPlan.length) return null;
317
+ // When the plan is exhausted, wrap (over-generation due to noise rounding;
318
+ // bounded re-use keeps remaining events on picked days rather than spilling).
319
+ const dayStartSec = dayPlan[dayPlanCursor % dayPlan.length];
320
+ dayPlanCursor++;
321
+ const dayEndSec = Math.min(dayStartSec + 86400 - 1, context.FIXED_NOW);
322
+ return { earliest: Math.max(dayStartSec, context.FIXED_BEGIN), latest: dayEndSec };
323
+ };
324
+
284
325
  const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
285
326
  .filter((f) => !f.conditions || matchConditions(profile, f.conditions))
286
327
  .reduce(weighFunnels, []);
@@ -320,7 +361,16 @@ export async function userLoop(context) {
320
361
  // PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
321
362
  if (firstFunnels.length && userIsBornInDataset) {
322
363
  const firstFunnel = chance.pickone(firstFunnels, user);
323
- let cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
364
+ // Active-day mode: anchor the first funnel on a picked day so the user's
365
+ // signup lands within their planned active window. Legacy mode: anchor at
366
+ // adjustedCreated minus a noise offset.
367
+ let cursor;
368
+ if (dayPlan) {
369
+ const bounds = nextDayBounds();
370
+ cursor = bounds ? bounds.earliest : adjustedCreated.subtract(noise(), 'seconds').unix();
371
+ } else {
372
+ cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
373
+ }
324
374
 
325
375
  // Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
326
376
  // passes = failedPriors + 1. Validator coerced bounds; default both 0.
@@ -391,20 +441,99 @@ export async function userLoop(context) {
391
441
  // Usage funnels for converted users: identity already stitched, just default 'both'.
392
442
  const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
393
443
 
394
- let usageFunnelCursor = userFirstEventTime;
395
- while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
444
+ // v1.5 follow-up (engine bunchiness fix, 2026-05-09):
445
+ // REMOVED `usageFunnelCursor` accumulator. Each funnel call now uses
446
+ // `userFirstEventTime` (constant) as the anchor. Without this, the cursor
447
+ // chained `last_event_time + small_gap` between funnel runs, walking past
448
+ // FIXED_NOW and producing the right-edge bunchiness regression. See
449
+ // `plans/ENGINE-BUNCHINESS/FIX.md` for the full diagnosis.
450
+ //
451
+ // Loop budget now counts SURVIVING events (post `_drop`), not raw output.
452
+ // Combined with the cursor removal, this gives each funnel an independent
453
+ // uniform-in-window anchor (via TimeSoup) and lets the budget loop iterate
454
+ // until the user actually has the target event count.
455
+ //
456
+ // v1.5: when auto-promote marks every event strict (e.g. all events appear in
457
+ // the only funnel), there's nothing for the standalone branch to pick. Bail
458
+ // out cleanly rather than crashing on `pick([])`.
459
+ const hasUsageFunnels = usageFunnels.length > 0;
460
+ const hasStandaloneEvents = weightedEvents.length > 0;
461
+ // Hard ceiling on iterations: defends against pathological configs where
462
+ // every funnel produces 0 surviving events (e.g. ttc > numDays so step1
463
+ // uniform anchor always lands too late). Cap at 2× expected iteration count
464
+ // based on average funnel length (~7 steps) to avoid infinite loops.
465
+ const MAX_ITERATIONS = Math.max(100, numEventsThisUserWillPreform * 2);
466
+ let iterationCount = 0;
467
+ while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled && iterationCount < MAX_ITERATIONS) {
468
+ iterationCount++;
396
469
  let newEvents;
397
- if (usageFunnels.length && userConverted) {
470
+ // v1.5 active-day: pop next picked-day bounds. Pass `latestTime` through
471
+ // featureCtx so makeEvent's TimeSoup confines to that day. Funnel cursor
472
+ // gets re-anchored to the picked day's start (subsequent funnel steps
473
+ // spill within `timeToConvert` hours; this is intentional).
474
+ const dayBounds = dayPlan ? nextDayBounds() : null;
475
+ // Compute step1's `latestTime` so the funnel's relative span fits before
476
+ // FIXED_NOW. Without this, born-late users + long-ttc funnels generate
477
+ // large numbers of `_drop`'d events that consume budget cycles. The
478
+ // safety margin is `timeToConvert * 3600 - 1` seconds (matches v1.5
479
+ // conversion window contract).
480
+ const standaloneFeatureCtx = dayBounds
481
+ ? { ...featureCtx, latestTime: dayBounds.latest }
482
+ : featureCtx;
483
+ // Nothing to generate: user converted via firstFunnel only and has no
484
+ // usage funnels and no standalone events. Stop attempting.
485
+ if ((!hasUsageFunnels || !userConverted) && !hasStandaloneEvents) break;
486
+ if (hasUsageFunnels && userConverted) {
398
487
  const currentFunnel = chance.pickone(usageFunnels);
399
- const [data, converted] = await makeFunnel(context, currentFunnel, user, usageFunnelCursor, profile, userSCD, persona, featureCtx, usageAttemptMeta);
488
+ const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
489
+ // Anchor cursor at picked day's start when active-day mode is on,
490
+ // otherwise pass userFirstEventTime (constant). NO cursor accumulation.
491
+ const funnelCursor = dayBounds ? dayBounds.earliest : userFirstEventTime;
492
+ // Constrain funnel step1's TimeSoup latestTime so the full funnel fits in
493
+ // window. Without this, late steps spill past FIXED_NOW and get `_drop`'d.
494
+ //
495
+ // v1.5 final (2026-05-09): `FUNNEL_DEAD_ZONE_CAP_SEC = 0` — funnels can
496
+ // anchor step1 right up to FIXED_NOW. Earlier rounds defended against a
497
+ // cursor-accumulation bug by reserving a `ttc`-sized dead zone at the
498
+ // right edge; round 1 fixed cursor accumulation directly, leaving the
499
+ // dead zone as defense-in-depth. The future-time guard at storage step
500
+ // 14 (per CLAUDE.md "Execution Order") drops any event with `time >
501
+ // FIXED_NOW`, so spillover from late funnel steps is filtered there
502
+ // instead of by anchoring upstream. Removing the dead zone eliminated
503
+ // the last-day cliff for funnel-heavy dungeons WITHOUT re-introducing
504
+ // `futureEvents > 0` — verified across the 194-combo engine-validation
505
+ // sweep (`scripts/sweep-engine.mjs`, `plans/ENGINE-VALIDATION/FIX.md`).
506
+ //
507
+ // Trade-off retained: long-ttc funnels still lose some late steps to
508
+ // `_drop`. Budget loop iterates more to compensate. Catch-all funnel
509
+ // (`ttc=1d`, set in config-validator) is unaffected.
510
+ //
511
+ // Born-late edge case: when `funnelCursor > FN`, the `safeLatest >
512
+ // funnelCursor` check below falls back to `FN` so the user can emit.
513
+ const FUNNEL_DEAD_ZONE_CAP_SEC = 0;
514
+ const deadZoneSec = Math.min(ttcSec, FUNNEL_DEAD_ZONE_CAP_SEC);
515
+ const safeLatest = context.FIXED_NOW - deadZoneSec;
516
+ const funnelLatestTime = dayBounds
517
+ ? dayBounds.latest
518
+ : (safeLatest > funnelCursor ? safeLatest : context.FIXED_NOW);
519
+ const funnelEventFeatureCtx = { ...featureCtx, latestTime: funnelLatestTime };
520
+ const [data, converted] = await makeFunnel(context, currentFunnel, user, funnelCursor, profile, userSCD, persona, funnelEventFeatureCtx, usageAttemptMeta);
521
+ // Budget counts raw output (matches pre-fix semantics). For short-ttc
522
+ // funnels (≤1d), almost no events `_drop` so the loop terminates at the
523
+ // expected count. For long-ttc funnels (>1d), some late steps `_drop` —
524
+ // loop iterates more to compensate, world-event `_drop`s still reduce
525
+ // the user's surviving total since they fire on dropped events too.
400
526
  numEventsPreformed += data.length;
401
527
  newEvents = data;
402
- if (data.length) {
403
- const lastTime = dayjs(data[data.length - 1].time).unix();
404
- usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
405
- }
406
528
  } else {
407
- const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, true, false, featureCtx, standaloneIdentityCtx);
529
+ // Active-day mode: standalone event uses picked-day start as earliestTime
530
+ // + day-end as latestTime (passed via eventFeatureCtx).
531
+ // `isFirstEvent: false` (8th arg) so TimeSoup distributes the timestamp
532
+ // — passing `true` here would pin every standalone event to the same
533
+ // `earliestTime`, which the now-deleted bunchIntoSessions used to paper
534
+ // over. With bunchIntoSessions removed, TimeSoup is the time source.
535
+ const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
536
+ const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
408
537
  numEventsPreformed++;
409
538
  newEvents = [data];
410
539
  }
@@ -434,7 +563,7 @@ export async function userLoop(context) {
434
563
  const userDecay = persona?.engagementDecay || globalEngagementDecay;
435
564
  if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
436
565
  // adjustedCreated and event times now share the same dataset window — no shift.
437
- usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
566
+ usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance, pickedDayBuckets);
438
567
  }
439
568
 
440
569
  // Feature 4: Data quality — duplicates and late-arriving
@@ -460,18 +589,17 @@ export async function userLoop(context) {
460
589
  }
461
590
  }
462
591
 
463
- // Session clustering: redistribute events into temporal bursts, then assign session IDs
592
+ // Session clustering: assign session IDs based on natural temporal gaps.
593
+ // v1.5: bunchIntoSessions deleted — was a redundant wholesale time-overwrite
594
+ // that scrambled multi-step funnels (round-robin into anchor buckets) and
595
+ // clobbered v1.5 active-day picking. assignSessionIds operates on the original
596
+ // TimeSoup-driven timestamps (Mixpanel-aligned 30-min-gap rule).
464
597
  if (hasSessionIds && usersEvents.length > 0) {
465
- const soupCfg = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
466
- const defaultPeaks = Math.max(5, (config.numDays || 30) * 2);
467
- const { mean: soupMean = 0, deviation: soupDev = 2, peaks: soupPeaks = defaultPeaks,
468
- dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD } = soupCfg;
469
-
470
- u.bunchIntoSessions(usersEvents, sessionTimeout, {
471
- earliestTime: userFirstEventTime,
472
- latestTime: context.FIXED_NOW,
473
- peaks: soupPeaks, deviation: soupDev, mean: soupMean,
474
- dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
598
+ // assignSessionIds requires events sorted ascending by time.
599
+ usersEvents.sort((a, b) => {
600
+ const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
601
+ const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
602
+ return ta - tb;
475
603
  });
476
604
  u.assignSessionIds(usersEvents, sessionTimeout);
477
605
 
@@ -493,6 +621,15 @@ export async function userLoop(context) {
493
621
  }
494
622
  }
495
623
 
624
+ // v1.5: Touchpoint cap. Sample up to `maxTouchpointsPerUser` (default 10,
625
+ // matching Mixpanel `TOUCHPOINTS_LIMIT`) eligible events from the user's
626
+ // lifetime and stamp UTMs on the sample. Lifetime-distributed sampling
627
+ // preserves realistic touch shape — last-10-window attribution then
628
+ // gives meaningful first/last-touch results.
629
+ if (config.hasCampaigns && usersEvents.length > 0) {
630
+ applyTouchpointCap(usersEvents, config, defaults, chance);
631
+ }
632
+
496
633
  // Hook for processing all user events (hooks override everything)
497
634
  if (config.hook) {
498
635
  // `meta.isPreAuth(event)` predicate bound to this user's auth state.
@@ -521,14 +658,38 @@ export async function userLoop(context) {
521
658
  if (Array.isArray(newEvents)) usersEvents = newEvents;
522
659
  }
523
660
 
661
+ // v1.5: auto-sort by time after everything hook. Defends against the
662
+ // most common new footgun — hooks that push() cloned events with
663
+ // arbitrary timestamps and break the greedy funnel engine's
664
+ // chronological-order requirement. Opt out with `autoSortAfterEverything: false`.
665
+ if (config.autoSortAfterEverything !== false && usersEvents.length > 1) {
666
+ usersEvents.sort((a, b) => {
667
+ const ta = (a && typeof a.time === 'string') ? Date.parse(a.time) : Number(a && a.time);
668
+ const tb = (b && typeof b.time === 'string') ? Date.parse(b.time) : Number(b && b.time);
669
+ if (!Number.isFinite(ta) && !Number.isFinite(tb)) return 0;
670
+ if (!Number.isFinite(ta)) return 1;
671
+ if (!Number.isFinite(tb)) return -1;
672
+ return ta - tb;
673
+ });
674
+ }
675
+
524
676
  // Defensive guard: drop any events whose timestamp landed past the
525
677
  // configured dataset end. Hooks that duplicate events with time offsets
526
678
  // (weekend surges, viral spreads) can leak a few past the boundary.
679
+ // v1.5 follow-up (`reccomendations-agent-1.md` Fix #2): surface the drop
680
+ // in verbose mode. Silent dropping made determinism failures hard to debug
681
+ // — events vanished without trace. Per-user log helps the owner see which
682
+ // users + how many events were affected.
683
+ const beforeFutureFilter = usersEvents.length;
527
684
  usersEvents = usersEvents.filter(e => {
528
685
  if (!e || !e.time) return true;
529
686
  const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
530
687
  return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
531
688
  });
689
+ const droppedFuture = beforeFutureFilter - usersEvents.length;
690
+ if (droppedFuture > 0 && config.verbose) {
691
+ console.warn(`⚠️ Dropped ${droppedFuture} future-dated event(s) for user ${distinct_id}`);
692
+ }
532
693
 
533
694
  // Store all user data
534
695
  await userProfilesData.hookPush(profile);
@@ -562,9 +723,11 @@ export async function userLoop(context) {
562
723
  if (dataQuality && dataQuality.botUsers > 0) {
563
724
  await generateBotUsers(context, dataQuality, storage);
564
725
  }
565
-
566
- // Clean up SIGINT handler
567
- process.removeListener('SIGINT', onSigint);
726
+ } finally {
727
+ // Always remove the SIGINT listener — even if userLoop throws or is
728
+ // cancelled. Pre-fix this leaked across test runs and stalled workers.
729
+ process.removeListener('SIGINT', onSigint);
730
+ }
568
731
  }
569
732
 
570
733
 
@@ -583,12 +746,261 @@ export function matchConditions(profile, conditions) {
583
746
  return true;
584
747
  }
585
748
 
749
+ // ── v1.5 Active-Day Plan Helpers ──
750
+
751
+ /**
752
+ * Build a deterministic per-user "day plan" for active-day mode.
753
+ *
754
+ * Each element of `plan` names the day on which the event-i should land. Events
755
+ * naturally concentrate onto `targetActiveDays` distinct days (drawn from a normal
756
+ * around `avgActiveDaysPerUser`, clamped to `[1, userActiveDays]`).
757
+ *
758
+ * Day picking uses weighted-without-replacement against soup DOW weights so the
759
+ * cohort-level weekly rhythm is preserved. Event distribution across picked days
760
+ * is proportional to those same weights, with a floor of 1 event per picked day.
761
+ *
762
+ * **Return shape (v1.5 follow-up — `reccomendations-agent-1.md` Fix #1):**
763
+ * `{ plan, pickedDayBuckets }` — `plan` is the shuffled per-event day-start
764
+ * unix-seconds array; `pickedDayBuckets` is a `Set<number>` of UTC-day-index
765
+ * buckets (`Math.floor(timestamp_ms / 86400000)`). Downstream consumers like
766
+ * `applyEngagementDecay` use `pickedDayBuckets` to enforce the v1.5
767
+ * distinct-day contract — see Fix #1 below.
768
+ *
769
+ * @param {Object} args
770
+ * @param {import('dayjs').Dayjs} args.adjustedCreated - User's "created" timestamp
771
+ * @param {number} args.fixedBegin - Dataset start (unix seconds)
772
+ * @param {number} args.fixedNow - Dataset end (unix seconds)
773
+ * @param {number} args.avgActiveDaysPerUser
774
+ * @param {number} args.userActiveDays - Capacity (max possible distinct days)
775
+ * @param {number} args.numEvents - Total events to schedule for this user
776
+ * @param {number[]} [args.dowWeights] - 7-element soup DOW weights (Sun..Sat)
777
+ * @param {Object} args.chance - Seeded chance instance
778
+ * @returns {{ plan: number[], pickedDayBuckets: Set<number> } | null} Day plan
779
+ * + bucket set, or null if not applicable (no candidate days, no events, etc.).
780
+ */
781
+ function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, chance }) {
782
+ if (!Number.isFinite(numEvents) || numEvents <= 0) return null;
783
+
784
+ // Candidate day buckets: UTC days intersecting [max(adjustedCreated, FIXED_BEGIN), FIXED_NOW].
785
+ // Pre-existing users with 'uniform' preExistingSpread have adjustedCreated < FIXED_BEGIN;
786
+ // active-day mode constrains them to in-window days only (matches what Mixpanel sees).
787
+ const userStartUnix = Math.max(
788
+ Math.floor(adjustedCreated.unix()),
789
+ Math.floor(fixedBegin)
790
+ );
791
+ const userEndUnix = Math.floor(fixedNow);
792
+ if (userEndUnix < userStartUnix) return null;
793
+
794
+ const dayMs = 86400;
795
+ const firstDay = Math.floor(userStartUnix / dayMs);
796
+ const lastDay = Math.floor(userEndUnix / dayMs);
797
+ const candidateDays = [];
798
+ for (let d = firstDay; d <= lastDay; d++) {
799
+ candidateDays.push(d * dayMs);
800
+ }
801
+ if (!candidateDays.length) return null;
802
+
803
+ // Weight by soup DOW (Sun=0..Sat=6). Default uniform if no weights configured.
804
+ const weights = candidateDays.map(daySec => {
805
+ if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
806
+ const dow = new Date(daySec * 1000).getUTCDay();
807
+ const w = Number(dowWeights[dow]);
808
+ return Number.isFinite(w) && w > 0 ? w : 0.0001;
809
+ });
810
+
811
+ // Draw target active-day count: normal(mean, sd=mean/3), clamped.
812
+ const meanActive = Math.max(1, Math.min(avgActiveDaysPerUser, candidateDays.length));
813
+ const sd = Math.max(0.5, meanActive / 3);
814
+ let targetActiveDays = Math.round(chance.normal({ mean: meanActive, dev: sd }));
815
+ targetActiveDays = Math.max(1, Math.min(targetActiveDays, candidateDays.length));
816
+
817
+ // Pick which days are active.
818
+ const pickedDays = weightedSampleNoReplacement(candidateDays, weights, targetActiveDays, chance);
819
+ if (!pickedDays.length) return null;
820
+ pickedDays.sort((a, b) => a - b);
821
+
822
+ // Recompute weights aligned to picked days for event distribution.
823
+ const pickedWeights = pickedDays.map(daySec => {
824
+ if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
825
+ const dow = new Date(daySec * 1000).getUTCDay();
826
+ const w = Number(dowWeights[dow]);
827
+ return Number.isFinite(w) && w > 0 ? w : 0.0001;
828
+ });
829
+ const totalWeight = pickedWeights.reduce((s, x) => s + x, 0);
830
+
831
+ // Allocate event counts across picked days. Each picked day gets at least 1
832
+ // event so every day is "used" — distinct-day count actually hits target.
833
+ const k = pickedDays.length;
834
+ let allocations;
835
+ if (numEvents <= k) {
836
+ // Fewer events than picked days: distribute one each, sample which days get them.
837
+ const subPicks = weightedSampleNoReplacement(pickedDays, pickedWeights, numEvents, chance);
838
+ const subSet = new Set(subPicks);
839
+ allocations = pickedDays.map(d => subSet.has(d) ? 1 : 0);
840
+ } else {
841
+ // Floor 1 per picked day, distribute remainder proportionally.
842
+ const remainder = numEvents - k;
843
+ allocations = pickedWeights.map(w => Math.floor((remainder * w) / totalWeight));
844
+ // Add floor of 1 to each
845
+ for (let i = 0; i < k; i++) allocations[i] += 1;
846
+ // Distribute rounding remainder by descending weight order
847
+ let assigned = allocations.reduce((s, x) => s + x, 0);
848
+ let rem = numEvents - assigned;
849
+ const order = pickedDays.map((_, i) => i).sort((a, b) => pickedWeights[b] - pickedWeights[a]);
850
+ for (let i = 0; i < rem; i++) {
851
+ allocations[order[i % order.length]]++;
852
+ }
853
+ }
854
+
855
+ // Expand into a flat plan and shuffle deterministically.
856
+ const plan = [];
857
+ for (let i = 0; i < k; i++) {
858
+ for (let j = 0; j < allocations[i]; j++) {
859
+ plan.push(pickedDays[i]);
860
+ }
861
+ }
862
+ // v1.5 follow-up (Fix #1): expose the picked-day bucket set so engagement
863
+ // decay can protect the last surviving event per picked day. Buckets use
864
+ // the same `floor(ms / 86400000)` formula that decay applies to event times.
865
+ const pickedDayBuckets = new Set(pickedDays.map(daySec => Math.floor(daySec / 86400)));
866
+ // chance.shuffle uses seeded RNG → deterministic.
867
+ return { plan: chance.shuffle(plan), pickedDayBuckets };
868
+ }
869
+
870
+ /**
871
+ * v1.5 Touchpoint cap: sample up to `maxTouchpointsPerUser` eligible events from
872
+ * the user's stream and stamp UTMs on the sample. Mirrors Mixpanel's
873
+ * `TOUCHPOINTS_LIMIT = 10` (`backend/libquery/properties_over_time/attributed_value_reader.cpp`).
874
+ *
875
+ * Eligibility:
876
+ * - If any event in `config.events[]` has `isAttributionEvent: true`: candidate
877
+ * pool = events whose name carries that flag.
878
+ * - Else (legacy fallback): candidate pool = ALL events in the user's stream.
879
+ *
880
+ * Sampling:
881
+ * - If `eligible.length <= cap`: stamp all of them.
882
+ * - Else: uniform random sample of size `cap` (seeded `chance.pickset`,
883
+ * deterministic). Sort sample chronologically before stamping so UTMs land
884
+ * in time order.
885
+ *
886
+ * Mutates `events` in place.
887
+ *
888
+ * @param {Object[]} events - User's events (flat shape with .event, .time)
889
+ * @param {Object} config - Validated dungeon config
890
+ * @param {Object} defaults - Context.defaults (provides campaigns())
891
+ * @param {Object} chance - Seeded chance instance
892
+ */
893
+ function applyTouchpointCap(events, config, defaults, chance) {
894
+ const cap = Number.isFinite(config.maxTouchpointsPerUser)
895
+ ? config.maxTouchpointsPerUser
896
+ : 10;
897
+ if (cap <= 0) return;
898
+
899
+ // Build event-name → config map for isAttributionEvent lookup.
900
+ const eventCfgByName = new Map();
901
+ for (const e of (config.events || [])) {
902
+ if (e && e.event) eventCfgByName.set(e.event, e);
903
+ }
904
+
905
+ // Determine eligible events.
906
+ let eligible;
907
+ if (config.hasAttributionFlags) {
908
+ eligible = events.filter(e => {
909
+ if (!e || !e.event) return false;
910
+ const cfg = eventCfgByName.get(e.event);
911
+ return cfg && cfg.isAttributionEvent === true;
912
+ });
913
+ } else {
914
+ eligible = events.slice();
915
+ }
916
+ if (eligible.length === 0) return;
917
+
918
+ // Sample up to cap (uniform random without replacement, seeded).
919
+ let sample;
920
+ if (eligible.length <= cap) {
921
+ sample = eligible;
922
+ } else {
923
+ sample = chance.pickset(eligible, cap);
924
+ }
925
+
926
+ // Sort sample chronologically so UTMs land in time order.
927
+ sample.sort((a, b) => {
928
+ const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
929
+ const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
930
+ return ta - tb;
931
+ });
932
+
933
+ // Stamp UTMs on each sampled event using a campaign template.
934
+ for (const ev of sample) {
935
+ const campaignTemplate = u.pickRandom(defaults.campaigns());
936
+ if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
937
+ for (const [k, v] of Object.entries(campaignTemplate)) {
938
+ ev[k] = u.choose(v);
939
+ }
940
+ }
941
+ }
942
+
943
+ /**
944
+ * Weighted-without-replacement sampler. Samples `count` items from `items` without
945
+ * replacement, with selection probability proportional to `weights`.
946
+ *
947
+ * @param {*[]} items
948
+ * @param {number[]} weights
949
+ * @param {number} count
950
+ * @param {Object} chance - Seeded chance instance
951
+ * @returns {*[]}
952
+ */
953
+ function weightedSampleNoReplacement(items, weights, count, chance) {
954
+ if (count >= items.length) return items.slice();
955
+ const pool = items.slice();
956
+ const w = weights.slice();
957
+ const result = [];
958
+ for (let i = 0; i < count; i++) {
959
+ const total = w.reduce((s, x) => s + x, 0);
960
+ let chosen;
961
+ if (total <= 0) {
962
+ chosen = chance.integer({ min: 0, max: pool.length - 1 });
963
+ } else {
964
+ let roll = chance.floating({ min: 0, max: total });
965
+ chosen = 0;
966
+ for (let j = 0; j < w.length; j++) {
967
+ roll -= w[j];
968
+ if (roll <= 0) { chosen = j; break; }
969
+ }
970
+ }
971
+ result.push(pool[chosen]);
972
+ pool.splice(chosen, 1);
973
+ w.splice(chosen, 1);
974
+ }
975
+ return result;
976
+ }
977
+
586
978
  // ── Advanced Feature Helper Functions ──
587
979
 
588
980
  /**
589
- * Feature 3: Apply engagement decay to a user's events
981
+ * Feature 3: Apply engagement decay to a user's events.
982
+ *
983
+ * Filters events probabilistically per the configured decay model
984
+ * (`exponential` / `linear` / `step`). Late events drop more often.
985
+ *
986
+ * **v1.5 follow-up (`reccomendations-agent-1.md` Fix #1):** when
987
+ * `pickedDayBuckets` is provided (active-day mode), the filter NEVER drops the
988
+ * last surviving event on any picked day. Without this, decay can silently
989
+ * undershoot the configured `avgActiveDaysPerUser` by killing all events on
990
+ * sparse late days (e.g., exponential decay with short half-life). The protect-
991
+ * last-event logic preserves the v1.5 distinct-day contract.
992
+ *
993
+ * @param {Object[]} events - User's events (mutated by filter; new array returned)
994
+ * @param {Object} decay - `engagementDecay` config (model/halfLife/floor/etc.)
995
+ * @param {*} userCreated - dayjs object or ISO string of user's creation time
996
+ * @param {Object} context - Engine context (unused; kept for future use)
997
+ * @param {Object} chance - Seeded chance instance
998
+ * @param {Set<number> | null} [pickedDayBuckets=null] - From `buildActiveDayPlan`.
999
+ * Each bucket = `Math.floor(timestamp_ms / 86400000)`. When non-null, the
1000
+ * filter protects the last surviving event on each bucket.
1001
+ * @returns {Object[]} Filtered events array
590
1002
  */
591
- function applyEngagementDecay(events, decay, userCreated, context, chance) {
1003
+ function applyEngagementDecay(events, decay, userCreated, context, chance, pickedDayBuckets = null) {
592
1004
  if (!events.length) return events;
593
1005
  // Perf 2: Use Date.parse instead of dayjs for hot loop
594
1006
  const userStartUnix = new Date(userCreated.toISOString ? userCreated.toISOString() : userCreated).getTime() / 1000;
@@ -597,6 +1009,23 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
597
1009
  const reactivationChance = decay.reactivationChance || 0;
598
1010
  const reactivationMult = decay.reactivationMultiplier || 2.0;
599
1011
 
1012
+ // v1.5 Fix #1: pre-compute per-bucket counts so we can enforce "at least one
1013
+ // event survives on each picked day". Only counts events that fall within a
1014
+ // picked-day bucket (events spilled to other days don't count toward the
1015
+ // per-bucket survivor budget).
1016
+ const dayMs = 86400000;
1017
+ const dayCounts = new Map(); // bucket → remaining (mutable) event count
1018
+ if (pickedDayBuckets && pickedDayBuckets.size) {
1019
+ for (const ev of events) {
1020
+ const t = new Date(ev.time).getTime();
1021
+ if (!Number.isFinite(t)) continue;
1022
+ const bucket = Math.floor(t / dayMs);
1023
+ if (pickedDayBuckets.has(bucket)) {
1024
+ dayCounts.set(bucket, (dayCounts.get(bucket) || 0) + 1);
1025
+ }
1026
+ }
1027
+ }
1028
+
600
1029
  return events.filter(ev => {
601
1030
  const evUnix = new Date(ev.time).getTime() / 1000;
602
1031
  const daysSinceBirth = Math.max(0, (evUnix - userStartUnix) / 86400);
@@ -621,7 +1050,27 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
621
1050
  retention = Math.min(1.0, retention * reactivationMult);
622
1051
  }
623
1052
 
624
- return chance.bool({ likelihood: retention * 100 });
1053
+ const keep = chance.bool({ likelihood: retention * 100 });
1054
+
1055
+ // v1.5 Fix #1: protect the last surviving event on each picked day.
1056
+ // Determinism preserved: no extra RNG calls, just a count check.
1057
+ if (pickedDayBuckets && pickedDayBuckets.size) {
1058
+ const bucket = Math.floor(new Date(ev.time).getTime() / dayMs);
1059
+ if (pickedDayBuckets.has(bucket)) {
1060
+ const remaining = dayCounts.get(bucket) || 0;
1061
+ if (!keep) {
1062
+ if (remaining <= 1) {
1063
+ // Last event on this picked day — protect it. Leave dayCounts
1064
+ // at 1 so subsequent events on this bucket can still drop.
1065
+ return true;
1066
+ }
1067
+ dayCounts.set(bucket, remaining - 1);
1068
+ }
1069
+ // If keep===true, no decrement — the survivor still counts.
1070
+ }
1071
+ }
1072
+
1073
+ return keep;
625
1074
  });
626
1075
  }
627
1076