@ak--47/dungeon-master 1.4.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +158 -0
- package/.claude/skills/create-dungeon/SKILL.md +464 -0
- package/.claude/skills/verify-dungeon/SKILL.md +157 -0
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
- package/.claude/skills/write-hooks/SKILL.md +468 -0
- package/CHANGELOG.md +139 -0
- package/HOOKS.md +1243 -597
- package/README.md +140 -5
- package/dungeons/technical/ad-spend.js +1 -1
- package/dungeons/technical/anonymous-users.js +1 -1
- package/dungeons/technical/array-of-object-lookup.js +1 -1
- package/dungeons/technical/datagen-v15-verify.js +74 -0
- package/dungeons/technical/experiments.js +1 -1
- package/dungeons/technical/foobar.js +1 -1
- package/dungeons/technical/group-analytics.js +1 -1
- package/dungeons/technical/mirror-strategies.js +1 -1
- package/dungeons/technical/nested-objects.js +1 -1
- package/dungeons/technical/retention-cadence.js +1 -1
- package/dungeons/technical/sanity.js +1 -1
- package/dungeons/technical/scale-test.js +1 -1
- package/dungeons/technical/scd.js +1 -1
- package/dungeons/technical/simple.js +1 -1
- package/dungeons/technical/simplest.js +74 -20
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +4 -0
- package/dungeons/vertical/community.js +9 -3
- package/dungeons/vertical/crypto.js +5 -0
- package/dungeons/vertical/dating.js +23 -10
- package/dungeons/vertical/devtools.js +10 -0
- package/dungeons/vertical/ecommerce.js +6 -0
- package/dungeons/vertical/education.js +11 -0
- package/dungeons/vertical/fintech.js +13 -0
- package/dungeons/vertical/fitness.js +10 -0
- package/dungeons/vertical/food-delivery.js +9 -0
- package/dungeons/vertical/gaming.js +10 -0
- package/dungeons/vertical/healthcare.js +5 -0
- package/dungeons/vertical/insurance-application.js +10 -0
- package/dungeons/vertical/logistics.js +8 -1
- package/dungeons/vertical/marketplace.js +7 -0
- package/dungeons/vertical/media.js +8 -0
- package/dungeons/vertical/real-estate.js +7 -1
- package/dungeons/vertical/sass.js +12 -0
- package/dungeons/vertical/social.js +9 -0
- package/dungeons/vertical/travel.js +5 -0
- package/index.js +19 -4
- package/lib/core/config-validator.js +270 -7
- package/lib/core/dungeon-loader.js +2 -5
- package/lib/generators/events.js +12 -13
- package/lib/generators/funnels.js +72 -1
- package/lib/hook-helpers/index.js +1 -0
- package/lib/hook-helpers/inject.js +95 -0
- package/lib/orchestrators/user-loop.js +478 -29
- package/lib/templates/macro-presets.js +39 -9
- package/lib/utils/utils.js +16 -79
- package/lib/verify/counting.js +320 -0
- package/lib/verify/emulate-breakdown.js +512 -108
- package/lib/verify/funnel-engine.js +539 -0
- package/lib/verify/identity.js +78 -0
- package/lib/verify/index.js +19 -0
- package/lib/verify/verify-dungeon.js +58 -0
- package/package.json +4 -2
- package/types.d.ts +237 -4
- package/scripts/smoke-test-all.mjs +0 -162
|
@@ -42,6 +42,7 @@ export async function userLoop(context) {
|
|
|
42
42
|
scdProps,
|
|
43
43
|
numDays,
|
|
44
44
|
avgEventsPerUserPerDay,
|
|
45
|
+
avgActiveDaysPerUser,
|
|
45
46
|
percentUsersBornInDataset = 15,
|
|
46
47
|
strictEventCount = false,
|
|
47
48
|
bornRecentBias = 0, // -1..1; positive = births skew toward end of window
|
|
@@ -66,7 +67,10 @@ export async function userLoop(context) {
|
|
|
66
67
|
// Track if we've already logged the strict event count message
|
|
67
68
|
let hasLoggedStrictCountReached = false;
|
|
68
69
|
|
|
69
|
-
// Handle graceful shutdown on SIGINT (Ctrl+C)
|
|
70
|
+
// Handle graceful shutdown on SIGINT (Ctrl+C).
|
|
71
|
+
// CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
|
|
72
|
+
// cancellation paths leaked listeners across test runs, accumulating until
|
|
73
|
+
// Node fired the MaxListenersExceededWarning AND test workers stalled.
|
|
70
74
|
let cancelled = false;
|
|
71
75
|
const onSigint = () => {
|
|
72
76
|
cancelled = true;
|
|
@@ -75,6 +79,7 @@ export async function userLoop(context) {
|
|
|
75
79
|
};
|
|
76
80
|
process.on('SIGINT', onSigint);
|
|
77
81
|
|
|
82
|
+
try {
|
|
78
83
|
for (let i = 0; i < numUsers; i++) {
|
|
79
84
|
const userPromise = USER_CONN(async () => {
|
|
80
85
|
// Bail out if cancelled
|
|
@@ -281,6 +286,42 @@ export async function userLoop(context) {
|
|
|
281
286
|
|
|
282
287
|
let userFirstEventTime;
|
|
283
288
|
|
|
289
|
+
// ── v1.5 Active-day scheduling ──
|
|
290
|
+
// When `avgActiveDaysPerUser` is set, build a per-user day plan: a list
|
|
291
|
+
// of UTC day-start unix-seconds, one entry per planned event. Each event
|
|
292
|
+
// generation call pops the next day from the plan and constrains TimeSoup
|
|
293
|
+
// to that day's [start, end] range. Funnel events anchor on the picked
|
|
294
|
+
// day; subsequent funnel steps spill within `timeToConvert` hours.
|
|
295
|
+
//
|
|
296
|
+
// When unset, dayPlan stays null and behavior is fully legacy (TimeSoup
|
|
297
|
+
// across [adjustedCreated, FIXED_NOW]).
|
|
298
|
+
const soupCfgForActiveDay = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
|
|
299
|
+
// v1.5 follow-up (Fix #1): buildActiveDayPlan returns
|
|
300
|
+
// `{ plan, pickedDayBuckets }`. `pickedDayBuckets` flows into
|
|
301
|
+
// `applyEngagementDecay` so the decay filter never drops the last event
|
|
302
|
+
// on a picked day — preserving the configured distinct-day count.
|
|
303
|
+
const dayPlanResult = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser))
|
|
304
|
+
? buildActiveDayPlan({
|
|
305
|
+
adjustedCreated, fixedBegin: context.FIXED_BEGIN, fixedNow: context.FIXED_NOW,
|
|
306
|
+
avgActiveDaysPerUser, userActiveDays,
|
|
307
|
+
numEvents: numEventsThisUserWillPreform,
|
|
308
|
+
dowWeights: soupCfgForActiveDay.dayOfWeekWeights,
|
|
309
|
+
chance,
|
|
310
|
+
})
|
|
311
|
+
: null;
|
|
312
|
+
const dayPlan = dayPlanResult ? dayPlanResult.plan : null;
|
|
313
|
+
const pickedDayBuckets = dayPlanResult ? dayPlanResult.pickedDayBuckets : null;
|
|
314
|
+
let dayPlanCursor = 0;
|
|
315
|
+
const nextDayBounds = () => {
|
|
316
|
+
if (!dayPlan || !dayPlan.length) return null;
|
|
317
|
+
// When the plan is exhausted, wrap (over-generation due to noise rounding;
|
|
318
|
+
// bounded re-use keeps remaining events on picked days rather than spilling).
|
|
319
|
+
const dayStartSec = dayPlan[dayPlanCursor % dayPlan.length];
|
|
320
|
+
dayPlanCursor++;
|
|
321
|
+
const dayEndSec = Math.min(dayStartSec + 86400 - 1, context.FIXED_NOW);
|
|
322
|
+
return { earliest: Math.max(dayStartSec, context.FIXED_BEGIN), latest: dayEndSec };
|
|
323
|
+
};
|
|
324
|
+
|
|
284
325
|
const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
|
|
285
326
|
.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
|
|
286
327
|
.reduce(weighFunnels, []);
|
|
@@ -320,7 +361,16 @@ export async function userLoop(context) {
|
|
|
320
361
|
// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
|
|
321
362
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
322
363
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
323
|
-
|
|
364
|
+
// Active-day mode: anchor the first funnel on a picked day so the user's
|
|
365
|
+
// signup lands within their planned active window. Legacy mode: anchor at
|
|
366
|
+
// adjustedCreated minus a noise offset.
|
|
367
|
+
let cursor;
|
|
368
|
+
if (dayPlan) {
|
|
369
|
+
const bounds = nextDayBounds();
|
|
370
|
+
cursor = bounds ? bounds.earliest : adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
371
|
+
} else {
|
|
372
|
+
cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
373
|
+
}
|
|
324
374
|
|
|
325
375
|
// Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
|
|
326
376
|
// passes = failedPriors + 1. Validator coerced bounds; default both 0.
|
|
@@ -391,20 +441,99 @@ export async function userLoop(context) {
|
|
|
391
441
|
// Usage funnels for converted users: identity already stitched, just default 'both'.
|
|
392
442
|
const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
|
|
393
443
|
|
|
394
|
-
|
|
395
|
-
|
|
444
|
+
// v1.5 follow-up (engine bunchiness fix, 2026-05-09):
|
|
445
|
+
// REMOVED `usageFunnelCursor` accumulator. Each funnel call now uses
|
|
446
|
+
// `userFirstEventTime` (constant) as the anchor. Without this, the cursor
|
|
447
|
+
// chained `last_event_time + small_gap` between funnel runs, walking past
|
|
448
|
+
// FIXED_NOW and producing the right-edge bunchiness regression. See
|
|
449
|
+
// `plans/ENGINE-BUNCHINESS/FIX.md` for the full diagnosis.
|
|
450
|
+
//
|
|
451
|
+
// Loop budget now counts SURVIVING events (post `_drop`), not raw output.
|
|
452
|
+
// Combined with the cursor removal, this gives each funnel an independent
|
|
453
|
+
// uniform-in-window anchor (via TimeSoup) and lets the budget loop iterate
|
|
454
|
+
// until the user actually has the target event count.
|
|
455
|
+
//
|
|
456
|
+
// v1.5: when auto-promote marks every event strict (e.g. all events appear in
|
|
457
|
+
// the only funnel), there's nothing for the standalone branch to pick. Bail
|
|
458
|
+
// out cleanly rather than crashing on `pick([])`.
|
|
459
|
+
const hasUsageFunnels = usageFunnels.length > 0;
|
|
460
|
+
const hasStandaloneEvents = weightedEvents.length > 0;
|
|
461
|
+
// Hard ceiling on iterations: defends against pathological configs where
|
|
462
|
+
// every funnel produces 0 surviving events (e.g. ttc > numDays so step1
|
|
463
|
+
// uniform anchor always lands too late). Cap at 2× expected iteration count
|
|
464
|
+
// based on average funnel length (~7 steps) to avoid infinite loops.
|
|
465
|
+
const MAX_ITERATIONS = Math.max(100, numEventsThisUserWillPreform * 2);
|
|
466
|
+
let iterationCount = 0;
|
|
467
|
+
while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled && iterationCount < MAX_ITERATIONS) {
|
|
468
|
+
iterationCount++;
|
|
396
469
|
let newEvents;
|
|
397
|
-
|
|
470
|
+
// v1.5 active-day: pop next picked-day bounds. Pass `latestTime` through
|
|
471
|
+
// featureCtx so makeEvent's TimeSoup confines to that day. Funnel cursor
|
|
472
|
+
// gets re-anchored to the picked day's start (subsequent funnel steps
|
|
473
|
+
// spill within `timeToConvert` hours; this is intentional).
|
|
474
|
+
const dayBounds = dayPlan ? nextDayBounds() : null;
|
|
475
|
+
// Compute step1's `latestTime` so the funnel's relative span fits before
|
|
476
|
+
// FIXED_NOW. Without this, born-late users + long-ttc funnels generate
|
|
477
|
+
// large numbers of `_drop`'d events that consume budget cycles. The
|
|
478
|
+
// safety margin is `timeToConvert * 3600 - 1` seconds (matches v1.5
|
|
479
|
+
// conversion window contract).
|
|
480
|
+
const standaloneFeatureCtx = dayBounds
|
|
481
|
+
? { ...featureCtx, latestTime: dayBounds.latest }
|
|
482
|
+
: featureCtx;
|
|
483
|
+
// Nothing to generate: user converted via firstFunnel only and has no
|
|
484
|
+
// usage funnels and no standalone events. Stop attempting.
|
|
485
|
+
if ((!hasUsageFunnels || !userConverted) && !hasStandaloneEvents) break;
|
|
486
|
+
if (hasUsageFunnels && userConverted) {
|
|
398
487
|
const currentFunnel = chance.pickone(usageFunnels);
|
|
399
|
-
const
|
|
488
|
+
const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
|
|
489
|
+
// Anchor cursor at picked day's start when active-day mode is on,
|
|
490
|
+
// otherwise pass userFirstEventTime (constant). NO cursor accumulation.
|
|
491
|
+
const funnelCursor = dayBounds ? dayBounds.earliest : userFirstEventTime;
|
|
492
|
+
// Constrain funnel step1's TimeSoup latestTime so the full funnel fits in
|
|
493
|
+
// window. Without this, late steps spill past FIXED_NOW and get `_drop`'d.
|
|
494
|
+
//
|
|
495
|
+
// v1.5 final (2026-05-09): `FUNNEL_DEAD_ZONE_CAP_SEC = 0` — funnels can
|
|
496
|
+
// anchor step1 right up to FIXED_NOW. Earlier rounds defended against a
|
|
497
|
+
// cursor-accumulation bug by reserving a `ttc`-sized dead zone at the
|
|
498
|
+
// right edge; round 1 fixed cursor accumulation directly, leaving the
|
|
499
|
+
// dead zone as defense-in-depth. The future-time guard at storage step
|
|
500
|
+
// 14 (per CLAUDE.md "Execution Order") drops any event with `time >
|
|
501
|
+
// FIXED_NOW`, so spillover from late funnel steps is filtered there
|
|
502
|
+
// instead of by anchoring upstream. Removing the dead zone eliminated
|
|
503
|
+
// the last-day cliff for funnel-heavy dungeons WITHOUT re-introducing
|
|
504
|
+
// `futureEvents > 0` — verified across the 194-combo engine-validation
|
|
505
|
+
// sweep (`scripts/sweep-engine.mjs`, `plans/ENGINE-VALIDATION/FIX.md`).
|
|
506
|
+
//
|
|
507
|
+
// Trade-off retained: long-ttc funnels still lose some late steps to
|
|
508
|
+
// `_drop`. Budget loop iterates more to compensate. Catch-all funnel
|
|
509
|
+
// (`ttc=1d`, set in config-validator) is unaffected.
|
|
510
|
+
//
|
|
511
|
+
// Born-late edge case: when `funnelCursor > FN`, the `safeLatest >
|
|
512
|
+
// funnelCursor` check below falls back to `FN` so the user can emit.
|
|
513
|
+
const FUNNEL_DEAD_ZONE_CAP_SEC = 0;
|
|
514
|
+
const deadZoneSec = Math.min(ttcSec, FUNNEL_DEAD_ZONE_CAP_SEC);
|
|
515
|
+
const safeLatest = context.FIXED_NOW - deadZoneSec;
|
|
516
|
+
const funnelLatestTime = dayBounds
|
|
517
|
+
? dayBounds.latest
|
|
518
|
+
: (safeLatest > funnelCursor ? safeLatest : context.FIXED_NOW);
|
|
519
|
+
const funnelEventFeatureCtx = { ...featureCtx, latestTime: funnelLatestTime };
|
|
520
|
+
const [data, converted] = await makeFunnel(context, currentFunnel, user, funnelCursor, profile, userSCD, persona, funnelEventFeatureCtx, usageAttemptMeta);
|
|
521
|
+
// Budget counts raw output (matches pre-fix semantics). For short-ttc
|
|
522
|
+
// funnels (≤1d), almost no events `_drop` so the loop terminates at the
|
|
523
|
+
// expected count. For long-ttc funnels (>1d), some late steps `_drop` —
|
|
524
|
+
// loop iterates more to compensate, world-event `_drop`s still reduce
|
|
525
|
+
// the user's surviving total since they fire on dropped events too.
|
|
400
526
|
numEventsPreformed += data.length;
|
|
401
527
|
newEvents = data;
|
|
402
|
-
if (data.length) {
|
|
403
|
-
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
404
|
-
usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
|
|
405
|
-
}
|
|
406
528
|
} else {
|
|
407
|
-
|
|
529
|
+
// Active-day mode: standalone event uses picked-day start as earliestTime
|
|
530
|
+
// + day-end as latestTime (passed via eventFeatureCtx).
|
|
531
|
+
// `isFirstEvent: false` (8th arg) so TimeSoup distributes the timestamp
|
|
532
|
+
// — passing `true` here would pin every standalone event to the same
|
|
533
|
+
// `earliestTime`, which the now-deleted bunchIntoSessions used to paper
|
|
534
|
+
// over. With bunchIntoSessions removed, TimeSoup is the time source.
|
|
535
|
+
const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
|
|
536
|
+
const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
|
|
408
537
|
numEventsPreformed++;
|
|
409
538
|
newEvents = [data];
|
|
410
539
|
}
|
|
@@ -434,7 +563,7 @@ export async function userLoop(context) {
|
|
|
434
563
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
435
564
|
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
436
565
|
// adjustedCreated and event times now share the same dataset window — no shift.
|
|
437
|
-
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
566
|
+
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance, pickedDayBuckets);
|
|
438
567
|
}
|
|
439
568
|
|
|
440
569
|
// Feature 4: Data quality — duplicates and late-arriving
|
|
@@ -460,18 +589,17 @@ export async function userLoop(context) {
|
|
|
460
589
|
}
|
|
461
590
|
}
|
|
462
591
|
|
|
463
|
-
// Session clustering:
|
|
592
|
+
// Session clustering: assign session IDs based on natural temporal gaps.
|
|
593
|
+
// v1.5: bunchIntoSessions deleted — was a redundant wholesale time-overwrite
|
|
594
|
+
// that scrambled multi-step funnels (round-robin into anchor buckets) and
|
|
595
|
+
// clobbered v1.5 active-day picking. assignSessionIds operates on the original
|
|
596
|
+
// TimeSoup-driven timestamps (Mixpanel-aligned 30-min-gap rule).
|
|
464
597
|
if (hasSessionIds && usersEvents.length > 0) {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
u.bunchIntoSessions(usersEvents, sessionTimeout, {
|
|
471
|
-
earliestTime: userFirstEventTime,
|
|
472
|
-
latestTime: context.FIXED_NOW,
|
|
473
|
-
peaks: soupPeaks, deviation: soupDev, mean: soupMean,
|
|
474
|
-
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
598
|
+
// assignSessionIds requires events sorted ascending by time.
|
|
599
|
+
usersEvents.sort((a, b) => {
|
|
600
|
+
const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
|
|
601
|
+
const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
|
|
602
|
+
return ta - tb;
|
|
475
603
|
});
|
|
476
604
|
u.assignSessionIds(usersEvents, sessionTimeout);
|
|
477
605
|
|
|
@@ -493,6 +621,15 @@ export async function userLoop(context) {
|
|
|
493
621
|
}
|
|
494
622
|
}
|
|
495
623
|
|
|
624
|
+
// v1.5: Touchpoint cap. Sample up to `maxTouchpointsPerUser` (default 10,
|
|
625
|
+
// matching Mixpanel `TOUCHPOINTS_LIMIT`) eligible events from the user's
|
|
626
|
+
// lifetime and stamp UTMs on the sample. Lifetime-distributed sampling
|
|
627
|
+
// preserves realistic touch shape — last-10-window attribution then
|
|
628
|
+
// gives meaningful first/last-touch results.
|
|
629
|
+
if (config.hasCampaigns && usersEvents.length > 0) {
|
|
630
|
+
applyTouchpointCap(usersEvents, config, defaults, chance);
|
|
631
|
+
}
|
|
632
|
+
|
|
496
633
|
// Hook for processing all user events (hooks override everything)
|
|
497
634
|
if (config.hook) {
|
|
498
635
|
// `meta.isPreAuth(event)` predicate bound to this user's auth state.
|
|
@@ -521,14 +658,38 @@ export async function userLoop(context) {
|
|
|
521
658
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
522
659
|
}
|
|
523
660
|
|
|
661
|
+
// v1.5: auto-sort by time after everything hook. Defends against the
|
|
662
|
+
// most common new footgun — hooks that push() cloned events with
|
|
663
|
+
// arbitrary timestamps and break the greedy funnel engine's
|
|
664
|
+
// chronological-order requirement. Opt out with `autoSortAfterEverything: false`.
|
|
665
|
+
if (config.autoSortAfterEverything !== false && usersEvents.length > 1) {
|
|
666
|
+
usersEvents.sort((a, b) => {
|
|
667
|
+
const ta = (a && typeof a.time === 'string') ? Date.parse(a.time) : Number(a && a.time);
|
|
668
|
+
const tb = (b && typeof b.time === 'string') ? Date.parse(b.time) : Number(b && b.time);
|
|
669
|
+
if (!Number.isFinite(ta) && !Number.isFinite(tb)) return 0;
|
|
670
|
+
if (!Number.isFinite(ta)) return 1;
|
|
671
|
+
if (!Number.isFinite(tb)) return -1;
|
|
672
|
+
return ta - tb;
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
|
|
524
676
|
// Defensive guard: drop any events whose timestamp landed past the
|
|
525
677
|
// configured dataset end. Hooks that duplicate events with time offsets
|
|
526
678
|
// (weekend surges, viral spreads) can leak a few past the boundary.
|
|
679
|
+
// v1.5 follow-up (`reccomendations-agent-1.md` Fix #2): surface the drop
|
|
680
|
+
// in verbose mode. Silent dropping made determinism failures hard to debug
|
|
681
|
+
// — events vanished without trace. Per-user log helps the owner see which
|
|
682
|
+
// users + how many events were affected.
|
|
683
|
+
const beforeFutureFilter = usersEvents.length;
|
|
527
684
|
usersEvents = usersEvents.filter(e => {
|
|
528
685
|
if (!e || !e.time) return true;
|
|
529
686
|
const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
|
|
530
687
|
return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
|
|
531
688
|
});
|
|
689
|
+
const droppedFuture = beforeFutureFilter - usersEvents.length;
|
|
690
|
+
if (droppedFuture > 0 && config.verbose) {
|
|
691
|
+
console.warn(`⚠️ Dropped ${droppedFuture} future-dated event(s) for user ${distinct_id}`);
|
|
692
|
+
}
|
|
532
693
|
|
|
533
694
|
// Store all user data
|
|
534
695
|
await userProfilesData.hookPush(profile);
|
|
@@ -562,9 +723,11 @@ export async function userLoop(context) {
|
|
|
562
723
|
if (dataQuality && dataQuality.botUsers > 0) {
|
|
563
724
|
await generateBotUsers(context, dataQuality, storage);
|
|
564
725
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
726
|
+
} finally {
|
|
727
|
+
// Always remove the SIGINT listener — even if userLoop throws or is
|
|
728
|
+
// cancelled. Pre-fix this leaked across test runs and stalled workers.
|
|
729
|
+
process.removeListener('SIGINT', onSigint);
|
|
730
|
+
}
|
|
568
731
|
}
|
|
569
732
|
|
|
570
733
|
|
|
@@ -583,12 +746,261 @@ export function matchConditions(profile, conditions) {
|
|
|
583
746
|
return true;
|
|
584
747
|
}
|
|
585
748
|
|
|
749
|
+
// ── v1.5 Active-Day Plan Helpers ──
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* Build a deterministic per-user "day plan" for active-day mode.
|
|
753
|
+
*
|
|
754
|
+
* Each element of `plan` names the day on which the event-i should land. Events
|
|
755
|
+
* naturally concentrate onto `targetActiveDays` distinct days (drawn from a normal
|
|
756
|
+
* around `avgActiveDaysPerUser`, clamped to `[1, userActiveDays]`).
|
|
757
|
+
*
|
|
758
|
+
* Day picking uses weighted-without-replacement against soup DOW weights so the
|
|
759
|
+
* cohort-level weekly rhythm is preserved. Event distribution across picked days
|
|
760
|
+
* is proportional to those same weights, with a floor of 1 event per picked day.
|
|
761
|
+
*
|
|
762
|
+
* **Return shape (v1.5 follow-up — `reccomendations-agent-1.md` Fix #1):**
|
|
763
|
+
* `{ plan, pickedDayBuckets }` — `plan` is the shuffled per-event day-start
|
|
764
|
+
* unix-seconds array; `pickedDayBuckets` is a `Set<number>` of UTC-day-index
|
|
765
|
+
* buckets (`Math.floor(timestamp_ms / 86400000)`). Downstream consumers like
|
|
766
|
+
* `applyEngagementDecay` use `pickedDayBuckets` to enforce the v1.5
|
|
767
|
+
* distinct-day contract — see Fix #1 below.
|
|
768
|
+
*
|
|
769
|
+
* @param {Object} args
|
|
770
|
+
* @param {import('dayjs').Dayjs} args.adjustedCreated - User's "created" timestamp
|
|
771
|
+
* @param {number} args.fixedBegin - Dataset start (unix seconds)
|
|
772
|
+
* @param {number} args.fixedNow - Dataset end (unix seconds)
|
|
773
|
+
* @param {number} args.avgActiveDaysPerUser
|
|
774
|
+
* @param {number} args.userActiveDays - Capacity (max possible distinct days)
|
|
775
|
+
* @param {number} args.numEvents - Total events to schedule for this user
|
|
776
|
+
* @param {number[]} [args.dowWeights] - 7-element soup DOW weights (Sun..Sat)
|
|
777
|
+
* @param {Object} args.chance - Seeded chance instance
|
|
778
|
+
* @returns {{ plan: number[], pickedDayBuckets: Set<number> } | null} Day plan
|
|
779
|
+
* + bucket set, or null if not applicable (no candidate days, no events, etc.).
|
|
780
|
+
*/
|
|
781
|
+
function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, chance }) {
|
|
782
|
+
if (!Number.isFinite(numEvents) || numEvents <= 0) return null;
|
|
783
|
+
|
|
784
|
+
// Candidate day buckets: UTC days intersecting [max(adjustedCreated, FIXED_BEGIN), FIXED_NOW].
|
|
785
|
+
// Pre-existing users with 'uniform' preExistingSpread have adjustedCreated < FIXED_BEGIN;
|
|
786
|
+
// active-day mode constrains them to in-window days only (matches what Mixpanel sees).
|
|
787
|
+
const userStartUnix = Math.max(
|
|
788
|
+
Math.floor(adjustedCreated.unix()),
|
|
789
|
+
Math.floor(fixedBegin)
|
|
790
|
+
);
|
|
791
|
+
const userEndUnix = Math.floor(fixedNow);
|
|
792
|
+
if (userEndUnix < userStartUnix) return null;
|
|
793
|
+
|
|
794
|
+
const dayMs = 86400;
|
|
795
|
+
const firstDay = Math.floor(userStartUnix / dayMs);
|
|
796
|
+
const lastDay = Math.floor(userEndUnix / dayMs);
|
|
797
|
+
const candidateDays = [];
|
|
798
|
+
for (let d = firstDay; d <= lastDay; d++) {
|
|
799
|
+
candidateDays.push(d * dayMs);
|
|
800
|
+
}
|
|
801
|
+
if (!candidateDays.length) return null;
|
|
802
|
+
|
|
803
|
+
// Weight by soup DOW (Sun=0..Sat=6). Default uniform if no weights configured.
|
|
804
|
+
const weights = candidateDays.map(daySec => {
|
|
805
|
+
if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
|
|
806
|
+
const dow = new Date(daySec * 1000).getUTCDay();
|
|
807
|
+
const w = Number(dowWeights[dow]);
|
|
808
|
+
return Number.isFinite(w) && w > 0 ? w : 0.0001;
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
// Draw target active-day count: normal(mean, sd=mean/3), clamped.
|
|
812
|
+
const meanActive = Math.max(1, Math.min(avgActiveDaysPerUser, candidateDays.length));
|
|
813
|
+
const sd = Math.max(0.5, meanActive / 3);
|
|
814
|
+
let targetActiveDays = Math.round(chance.normal({ mean: meanActive, dev: sd }));
|
|
815
|
+
targetActiveDays = Math.max(1, Math.min(targetActiveDays, candidateDays.length));
|
|
816
|
+
|
|
817
|
+
// Pick which days are active.
|
|
818
|
+
const pickedDays = weightedSampleNoReplacement(candidateDays, weights, targetActiveDays, chance);
|
|
819
|
+
if (!pickedDays.length) return null;
|
|
820
|
+
pickedDays.sort((a, b) => a - b);
|
|
821
|
+
|
|
822
|
+
// Recompute weights aligned to picked days for event distribution.
|
|
823
|
+
const pickedWeights = pickedDays.map(daySec => {
|
|
824
|
+
if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
|
|
825
|
+
const dow = new Date(daySec * 1000).getUTCDay();
|
|
826
|
+
const w = Number(dowWeights[dow]);
|
|
827
|
+
return Number.isFinite(w) && w > 0 ? w : 0.0001;
|
|
828
|
+
});
|
|
829
|
+
const totalWeight = pickedWeights.reduce((s, x) => s + x, 0);
|
|
830
|
+
|
|
831
|
+
// Allocate event counts across picked days. Each picked day gets at least 1
|
|
832
|
+
// event so every day is "used" — distinct-day count actually hits target.
|
|
833
|
+
const k = pickedDays.length;
|
|
834
|
+
let allocations;
|
|
835
|
+
if (numEvents <= k) {
|
|
836
|
+
// Fewer events than picked days: distribute one each, sample which days get them.
|
|
837
|
+
const subPicks = weightedSampleNoReplacement(pickedDays, pickedWeights, numEvents, chance);
|
|
838
|
+
const subSet = new Set(subPicks);
|
|
839
|
+
allocations = pickedDays.map(d => subSet.has(d) ? 1 : 0);
|
|
840
|
+
} else {
|
|
841
|
+
// Floor 1 per picked day, distribute remainder proportionally.
|
|
842
|
+
const remainder = numEvents - k;
|
|
843
|
+
allocations = pickedWeights.map(w => Math.floor((remainder * w) / totalWeight));
|
|
844
|
+
// Add floor of 1 to each
|
|
845
|
+
for (let i = 0; i < k; i++) allocations[i] += 1;
|
|
846
|
+
// Distribute rounding remainder by descending weight order
|
|
847
|
+
let assigned = allocations.reduce((s, x) => s + x, 0);
|
|
848
|
+
let rem = numEvents - assigned;
|
|
849
|
+
const order = pickedDays.map((_, i) => i).sort((a, b) => pickedWeights[b] - pickedWeights[a]);
|
|
850
|
+
for (let i = 0; i < rem; i++) {
|
|
851
|
+
allocations[order[i % order.length]]++;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// Expand into a flat plan and shuffle deterministically.
|
|
856
|
+
const plan = [];
|
|
857
|
+
for (let i = 0; i < k; i++) {
|
|
858
|
+
for (let j = 0; j < allocations[i]; j++) {
|
|
859
|
+
plan.push(pickedDays[i]);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
// v1.5 follow-up (Fix #1): expose the picked-day bucket set so engagement
|
|
863
|
+
// decay can protect the last surviving event per picked day. Buckets use
|
|
864
|
+
// the same `floor(ms / 86400000)` formula that decay applies to event times.
|
|
865
|
+
const pickedDayBuckets = new Set(pickedDays.map(daySec => Math.floor(daySec / 86400)));
|
|
866
|
+
// chance.shuffle uses seeded RNG → deterministic.
|
|
867
|
+
return { plan: chance.shuffle(plan), pickedDayBuckets };
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* v1.5 Touchpoint cap: sample up to `maxTouchpointsPerUser` eligible events from
|
|
872
|
+
* the user's stream and stamp UTMs on the sample. Mirrors Mixpanel's
|
|
873
|
+
* `TOUCHPOINTS_LIMIT = 10` (`backend/libquery/properties_over_time/attributed_value_reader.cpp`).
|
|
874
|
+
*
|
|
875
|
+
* Eligibility:
|
|
876
|
+
* - If any event in `config.events[]` has `isAttributionEvent: true`: candidate
|
|
877
|
+
* pool = events whose name carries that flag.
|
|
878
|
+
* - Else (legacy fallback): candidate pool = ALL events in the user's stream.
|
|
879
|
+
*
|
|
880
|
+
* Sampling:
|
|
881
|
+
* - If `eligible.length <= cap`: stamp all of them.
|
|
882
|
+
* - Else: uniform random sample of size `cap` (seeded `chance.pickset`,
|
|
883
|
+
* deterministic). Sort sample chronologically before stamping so UTMs land
|
|
884
|
+
* in time order.
|
|
885
|
+
*
|
|
886
|
+
* Mutates `events` in place.
|
|
887
|
+
*
|
|
888
|
+
* @param {Object[]} events - User's events (flat shape with .event, .time)
|
|
889
|
+
* @param {Object} config - Validated dungeon config
|
|
890
|
+
* @param {Object} defaults - Context.defaults (provides campaigns())
|
|
891
|
+
* @param {Object} chance - Seeded chance instance
|
|
892
|
+
*/
|
|
893
|
+
function applyTouchpointCap(events, config, defaults, chance) {
|
|
894
|
+
const cap = Number.isFinite(config.maxTouchpointsPerUser)
|
|
895
|
+
? config.maxTouchpointsPerUser
|
|
896
|
+
: 10;
|
|
897
|
+
if (cap <= 0) return;
|
|
898
|
+
|
|
899
|
+
// Build event-name → config map for isAttributionEvent lookup.
|
|
900
|
+
const eventCfgByName = new Map();
|
|
901
|
+
for (const e of (config.events || [])) {
|
|
902
|
+
if (e && e.event) eventCfgByName.set(e.event, e);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// Determine eligible events.
|
|
906
|
+
let eligible;
|
|
907
|
+
if (config.hasAttributionFlags) {
|
|
908
|
+
eligible = events.filter(e => {
|
|
909
|
+
if (!e || !e.event) return false;
|
|
910
|
+
const cfg = eventCfgByName.get(e.event);
|
|
911
|
+
return cfg && cfg.isAttributionEvent === true;
|
|
912
|
+
});
|
|
913
|
+
} else {
|
|
914
|
+
eligible = events.slice();
|
|
915
|
+
}
|
|
916
|
+
if (eligible.length === 0) return;
|
|
917
|
+
|
|
918
|
+
// Sample up to cap (uniform random without replacement, seeded).
|
|
919
|
+
let sample;
|
|
920
|
+
if (eligible.length <= cap) {
|
|
921
|
+
sample = eligible;
|
|
922
|
+
} else {
|
|
923
|
+
sample = chance.pickset(eligible, cap);
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// Sort sample chronologically so UTMs land in time order.
|
|
927
|
+
sample.sort((a, b) => {
|
|
928
|
+
const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
|
|
929
|
+
const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
|
|
930
|
+
return ta - tb;
|
|
931
|
+
});
|
|
932
|
+
|
|
933
|
+
// Stamp UTMs on each sampled event using a campaign template.
|
|
934
|
+
for (const ev of sample) {
|
|
935
|
+
const campaignTemplate = u.pickRandom(defaults.campaigns());
|
|
936
|
+
if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
|
|
937
|
+
for (const [k, v] of Object.entries(campaignTemplate)) {
|
|
938
|
+
ev[k] = u.choose(v);
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Weighted-without-replacement sampler. Samples `count` items from `items` without
|
|
945
|
+
* replacement, with selection probability proportional to `weights`.
|
|
946
|
+
*
|
|
947
|
+
* @param {*[]} items
|
|
948
|
+
* @param {number[]} weights
|
|
949
|
+
* @param {number} count
|
|
950
|
+
* @param {Object} chance - Seeded chance instance
|
|
951
|
+
* @returns {*[]}
|
|
952
|
+
*/
|
|
953
|
+
function weightedSampleNoReplacement(items, weights, count, chance) {
|
|
954
|
+
if (count >= items.length) return items.slice();
|
|
955
|
+
const pool = items.slice();
|
|
956
|
+
const w = weights.slice();
|
|
957
|
+
const result = [];
|
|
958
|
+
for (let i = 0; i < count; i++) {
|
|
959
|
+
const total = w.reduce((s, x) => s + x, 0);
|
|
960
|
+
let chosen;
|
|
961
|
+
if (total <= 0) {
|
|
962
|
+
chosen = chance.integer({ min: 0, max: pool.length - 1 });
|
|
963
|
+
} else {
|
|
964
|
+
let roll = chance.floating({ min: 0, max: total });
|
|
965
|
+
chosen = 0;
|
|
966
|
+
for (let j = 0; j < w.length; j++) {
|
|
967
|
+
roll -= w[j];
|
|
968
|
+
if (roll <= 0) { chosen = j; break; }
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
result.push(pool[chosen]);
|
|
972
|
+
pool.splice(chosen, 1);
|
|
973
|
+
w.splice(chosen, 1);
|
|
974
|
+
}
|
|
975
|
+
return result;
|
|
976
|
+
}
|
|
977
|
+
|
|
586
978
|
// ── Advanced Feature Helper Functions ──
|
|
587
979
|
|
|
588
980
|
/**
|
|
589
|
-
* Feature 3: Apply engagement decay to a user's events
|
|
981
|
+
* Feature 3: Apply engagement decay to a user's events.
|
|
982
|
+
*
|
|
983
|
+
* Filters events probabilistically per the configured decay model
|
|
984
|
+
* (`exponential` / `linear` / `step`). Late events drop more often.
|
|
985
|
+
*
|
|
986
|
+
* **v1.5 follow-up (`reccomendations-agent-1.md` Fix #1):** when
|
|
987
|
+
* `pickedDayBuckets` is provided (active-day mode), the filter NEVER drops the
|
|
988
|
+
* last surviving event on any picked day. Without this, decay can silently
|
|
989
|
+
* undershoot the configured `avgActiveDaysPerUser` by killing all events on
|
|
990
|
+
* sparse late days (e.g., exponential decay with short half-life). The protect-
|
|
991
|
+
* last-event logic preserves the v1.5 distinct-day contract.
|
|
992
|
+
*
|
|
993
|
+
* @param {Object[]} events - User's events (mutated by filter; new array returned)
|
|
994
|
+
* @param {Object} decay - `engagementDecay` config (model/halfLife/floor/etc.)
|
|
995
|
+
* @param {*} userCreated - dayjs object or ISO string of user's creation time
|
|
996
|
+
* @param {Object} context - Engine context (unused; kept for future use)
|
|
997
|
+
* @param {Object} chance - Seeded chance instance
|
|
998
|
+
* @param {Set<number> | null} [pickedDayBuckets=null] - From `buildActiveDayPlan`.
|
|
999
|
+
* Each bucket = `Math.floor(timestamp_ms / 86400000)`. When non-null, the
|
|
1000
|
+
* filter protects the last surviving event on each bucket.
|
|
1001
|
+
* @returns {Object[]} Filtered events array
|
|
590
1002
|
*/
|
|
591
|
-
function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
1003
|
+
function applyEngagementDecay(events, decay, userCreated, context, chance, pickedDayBuckets = null) {
|
|
592
1004
|
if (!events.length) return events;
|
|
593
1005
|
// Perf 2: Use Date.parse instead of dayjs for hot loop
|
|
594
1006
|
const userStartUnix = new Date(userCreated.toISOString ? userCreated.toISOString() : userCreated).getTime() / 1000;
|
|
@@ -597,6 +1009,23 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
597
1009
|
const reactivationChance = decay.reactivationChance || 0;
|
|
598
1010
|
const reactivationMult = decay.reactivationMultiplier || 2.0;
|
|
599
1011
|
|
|
1012
|
+
// v1.5 Fix #1: pre-compute per-bucket counts so we can enforce "at least one
|
|
1013
|
+
// event survives on each picked day". Only counts events that fall within a
|
|
1014
|
+
// picked-day bucket (events spilled to other days don't count toward the
|
|
1015
|
+
// per-bucket survivor budget).
|
|
1016
|
+
const dayMs = 86400000;
|
|
1017
|
+
const dayCounts = new Map(); // bucket → remaining (mutable) event count
|
|
1018
|
+
if (pickedDayBuckets && pickedDayBuckets.size) {
|
|
1019
|
+
for (const ev of events) {
|
|
1020
|
+
const t = new Date(ev.time).getTime();
|
|
1021
|
+
if (!Number.isFinite(t)) continue;
|
|
1022
|
+
const bucket = Math.floor(t / dayMs);
|
|
1023
|
+
if (pickedDayBuckets.has(bucket)) {
|
|
1024
|
+
dayCounts.set(bucket, (dayCounts.get(bucket) || 0) + 1);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|
|
600
1029
|
return events.filter(ev => {
|
|
601
1030
|
const evUnix = new Date(ev.time).getTime() / 1000;
|
|
602
1031
|
const daysSinceBirth = Math.max(0, (evUnix - userStartUnix) / 86400);
|
|
@@ -621,7 +1050,27 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
621
1050
|
retention = Math.min(1.0, retention * reactivationMult);
|
|
622
1051
|
}
|
|
623
1052
|
|
|
624
|
-
|
|
1053
|
+
const keep = chance.bool({ likelihood: retention * 100 });
|
|
1054
|
+
|
|
1055
|
+
// v1.5 Fix #1: protect the last surviving event on each picked day.
|
|
1056
|
+
// Determinism preserved: no extra RNG calls, just a count check.
|
|
1057
|
+
if (pickedDayBuckets && pickedDayBuckets.size) {
|
|
1058
|
+
const bucket = Math.floor(new Date(ev.time).getTime() / dayMs);
|
|
1059
|
+
if (pickedDayBuckets.has(bucket)) {
|
|
1060
|
+
const remaining = dayCounts.get(bucket) || 0;
|
|
1061
|
+
if (!keep) {
|
|
1062
|
+
if (remaining <= 1) {
|
|
1063
|
+
// Last event on this picked day — protect it. Leave dayCounts
|
|
1064
|
+
// at 1 so subsequent events on this bucket can still drop.
|
|
1065
|
+
return true;
|
|
1066
|
+
}
|
|
1067
|
+
dayCounts.set(bucket, remaining - 1);
|
|
1068
|
+
}
|
|
1069
|
+
// If keep===true, no decrement — the survivor still counts.
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
return keep;
|
|
625
1074
|
});
|
|
626
1075
|
}
|
|
627
1076
|
|