@ak--47/dungeon-master 1.4.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +158 -0
- package/.claude/skills/create-dungeon/SKILL.md +464 -0
- package/.claude/skills/verify-dungeon/SKILL.md +157 -0
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
- package/.claude/skills/write-hooks/SKILL.md +468 -0
- package/CHANGELOG.md +147 -0
- package/HOOKS.md +1243 -597
- package/README.md +140 -5
- package/dungeons/technical/ad-spend.js +1 -1
- package/dungeons/technical/anonymous-users.js +1 -1
- package/dungeons/technical/array-of-object-lookup.js +1 -1
- package/dungeons/technical/datagen-v15-verify.js +74 -0
- package/dungeons/technical/experiments.js +1 -1
- package/dungeons/technical/foobar.js +1 -1
- package/dungeons/technical/group-analytics.js +1 -1
- package/dungeons/technical/mirror-strategies.js +1 -1
- package/dungeons/technical/nested-objects.js +1 -1
- package/dungeons/technical/retention-cadence.js +1 -1
- package/dungeons/technical/sanity.js +1 -1
- package/dungeons/technical/scale-test.js +1 -1
- package/dungeons/technical/scd.js +1 -1
- package/dungeons/technical/simple.js +1 -1
- package/dungeons/technical/simplest.js +74 -20
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +4 -0
- package/dungeons/vertical/community.js +9 -3
- package/dungeons/vertical/crypto.js +5 -0
- package/dungeons/vertical/dating.js +23 -10
- package/dungeons/vertical/devtools.js +10 -0
- package/dungeons/vertical/ecommerce.js +6 -0
- package/dungeons/vertical/education.js +11 -0
- package/dungeons/vertical/fintech.js +13 -0
- package/dungeons/vertical/fitness.js +10 -0
- package/dungeons/vertical/food-delivery.js +9 -0
- package/dungeons/vertical/gaming.js +10 -0
- package/dungeons/vertical/healthcare.js +5 -0
- package/dungeons/vertical/insurance-application.js +10 -0
- package/dungeons/vertical/logistics.js +8 -1
- package/dungeons/vertical/marketplace.js +7 -0
- package/dungeons/vertical/media.js +8 -0
- package/dungeons/vertical/real-estate.js +7 -1
- package/dungeons/vertical/sass.js +12 -0
- package/dungeons/vertical/social.js +9 -0
- package/dungeons/vertical/travel.js +5 -0
- package/index.js +45 -7
- package/lib/core/config-validator.js +270 -7
- package/lib/core/context.js +58 -0
- package/lib/core/dungeon-loader.js +2 -5
- package/lib/generators/events.js +12 -13
- package/lib/generators/funnels.js +72 -1
- package/lib/hook-helpers/index.js +1 -0
- package/lib/hook-helpers/inject.js +95 -0
- package/lib/orchestrators/mixpanel-sender.js +27 -1
- package/lib/orchestrators/user-loop.js +488 -29
- package/lib/templates/macro-presets.js +39 -9
- package/lib/utils/utils.js +16 -79
- package/lib/verify/counting.js +320 -0
- package/lib/verify/emulate-breakdown.js +512 -108
- package/lib/verify/funnel-engine.js +539 -0
- package/lib/verify/identity.js +78 -0
- package/lib/verify/index.js +19 -0
- package/lib/verify/verify-dungeon.js +58 -0
- package/package.json +4 -2
- package/types.d.ts +314 -4
- package/scripts/smoke-test-all.mjs +0 -162
|
@@ -42,6 +42,7 @@ export async function userLoop(context) {
|
|
|
42
42
|
scdProps,
|
|
43
43
|
numDays,
|
|
44
44
|
avgEventsPerUserPerDay,
|
|
45
|
+
avgActiveDaysPerUser,
|
|
45
46
|
percentUsersBornInDataset = 15,
|
|
46
47
|
strictEventCount = false,
|
|
47
48
|
bornRecentBias = 0, // -1..1; positive = births skew toward end of window
|
|
@@ -66,7 +67,10 @@ export async function userLoop(context) {
|
|
|
66
67
|
// Track if we've already logged the strict event count message
|
|
67
68
|
let hasLoggedStrictCountReached = false;
|
|
68
69
|
|
|
69
|
-
// Handle graceful shutdown on SIGINT (Ctrl+C)
|
|
70
|
+
// Handle graceful shutdown on SIGINT (Ctrl+C).
|
|
71
|
+
// CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
|
|
72
|
+
// cancellation paths leaked listeners across test runs, accumulating until
|
|
73
|
+
// Node fired the MaxListenersExceededWarning AND test workers stalled.
|
|
70
74
|
let cancelled = false;
|
|
71
75
|
const onSigint = () => {
|
|
72
76
|
cancelled = true;
|
|
@@ -75,6 +79,7 @@ export async function userLoop(context) {
|
|
|
75
79
|
};
|
|
76
80
|
process.on('SIGINT', onSigint);
|
|
77
81
|
|
|
82
|
+
try {
|
|
78
83
|
for (let i = 0; i < numUsers; i++) {
|
|
79
84
|
const userPromise = USER_CONN(async () => {
|
|
80
85
|
// Bail out if cancelled
|
|
@@ -104,6 +109,16 @@ export async function userLoop(context) {
|
|
|
104
109
|
]);
|
|
105
110
|
}
|
|
106
111
|
|
|
112
|
+
context.reportProgress({
|
|
113
|
+
phase: "generation",
|
|
114
|
+
users: context.getUserCount(),
|
|
115
|
+
events: context.getEventCount(),
|
|
116
|
+
eps,
|
|
117
|
+
memory: memUsed,
|
|
118
|
+
elapsed: duration,
|
|
119
|
+
percentComplete: Math.min(100, Math.round((context.getUserCount() / numUsers) * 100))
|
|
120
|
+
});
|
|
121
|
+
|
|
107
122
|
const userId = chance.guid();
|
|
108
123
|
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
|
|
109
124
|
const { distinct_id, created } = user;
|
|
@@ -271,6 +286,42 @@ export async function userLoop(context) {
|
|
|
271
286
|
|
|
272
287
|
let userFirstEventTime;
|
|
273
288
|
|
|
289
|
+
// ── v1.5 Active-day scheduling ──
|
|
290
|
+
// When `avgActiveDaysPerUser` is set, build a per-user day plan: a list
|
|
291
|
+
// of UTC day-start unix-seconds, one entry per planned event. Each event
|
|
292
|
+
// generation call pops the next day from the plan and constrains TimeSoup
|
|
293
|
+
// to that day's [start, end] range. Funnel events anchor on the picked
|
|
294
|
+
// day; subsequent funnel steps spill within `timeToConvert` hours.
|
|
295
|
+
//
|
|
296
|
+
// When unset, dayPlan stays null and behavior is fully legacy (TimeSoup
|
|
297
|
+
// across [adjustedCreated, FIXED_NOW]).
|
|
298
|
+
const soupCfgForActiveDay = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
|
|
299
|
+
// v1.5 follow-up (Fix #1): buildActiveDayPlan returns
|
|
300
|
+
// `{ plan, pickedDayBuckets }`. `pickedDayBuckets` flows into
|
|
301
|
+
// `applyEngagementDecay` so the decay filter never drops the last event
|
|
302
|
+
// on a picked day — preserving the configured distinct-day count.
|
|
303
|
+
const dayPlanResult = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser))
|
|
304
|
+
? buildActiveDayPlan({
|
|
305
|
+
adjustedCreated, fixedBegin: context.FIXED_BEGIN, fixedNow: context.FIXED_NOW,
|
|
306
|
+
avgActiveDaysPerUser, userActiveDays,
|
|
307
|
+
numEvents: numEventsThisUserWillPreform,
|
|
308
|
+
dowWeights: soupCfgForActiveDay.dayOfWeekWeights,
|
|
309
|
+
chance,
|
|
310
|
+
})
|
|
311
|
+
: null;
|
|
312
|
+
const dayPlan = dayPlanResult ? dayPlanResult.plan : null;
|
|
313
|
+
const pickedDayBuckets = dayPlanResult ? dayPlanResult.pickedDayBuckets : null;
|
|
314
|
+
let dayPlanCursor = 0;
|
|
315
|
+
const nextDayBounds = () => {
|
|
316
|
+
if (!dayPlan || !dayPlan.length) return null;
|
|
317
|
+
// When the plan is exhausted, wrap (over-generation due to noise rounding;
|
|
318
|
+
// bounded re-use keeps remaining events on picked days rather than spilling).
|
|
319
|
+
const dayStartSec = dayPlan[dayPlanCursor % dayPlan.length];
|
|
320
|
+
dayPlanCursor++;
|
|
321
|
+
const dayEndSec = Math.min(dayStartSec + 86400 - 1, context.FIXED_NOW);
|
|
322
|
+
return { earliest: Math.max(dayStartSec, context.FIXED_BEGIN), latest: dayEndSec };
|
|
323
|
+
};
|
|
324
|
+
|
|
274
325
|
const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
|
|
275
326
|
.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
|
|
276
327
|
.reduce(weighFunnels, []);
|
|
@@ -310,7 +361,16 @@ export async function userLoop(context) {
|
|
|
310
361
|
// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
|
|
311
362
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
312
363
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
313
|
-
|
|
364
|
+
// Active-day mode: anchor the first funnel on a picked day so the user's
|
|
365
|
+
// signup lands within their planned active window. Legacy mode: anchor at
|
|
366
|
+
// adjustedCreated minus a noise offset.
|
|
367
|
+
let cursor;
|
|
368
|
+
if (dayPlan) {
|
|
369
|
+
const bounds = nextDayBounds();
|
|
370
|
+
cursor = bounds ? bounds.earliest : adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
371
|
+
} else {
|
|
372
|
+
cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
373
|
+
}
|
|
314
374
|
|
|
315
375
|
// Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
|
|
316
376
|
// passes = failedPriors + 1. Validator coerced bounds; default both 0.
|
|
@@ -381,20 +441,99 @@ export async function userLoop(context) {
|
|
|
381
441
|
// Usage funnels for converted users: identity already stitched, just default 'both'.
|
|
382
442
|
const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
|
|
383
443
|
|
|
384
|
-
|
|
385
|
-
|
|
444
|
+
// v1.5 follow-up (engine bunchiness fix, 2026-05-09):
|
|
445
|
+
// REMOVED `usageFunnelCursor` accumulator. Each funnel call now uses
|
|
446
|
+
// `userFirstEventTime` (constant) as the anchor. Without this, the cursor
|
|
447
|
+
// chained `last_event_time + small_gap` between funnel runs, walking past
|
|
448
|
+
// FIXED_NOW and producing the right-edge bunchiness regression. See
|
|
449
|
+
// `plans/ENGINE-BUNCHINESS/FIX.md` for the full diagnosis.
|
|
450
|
+
//
|
|
451
|
+
// Loop budget now counts SURVIVING events (post `_drop`), not raw output.
|
|
452
|
+
// Combined with the cursor removal, this gives each funnel an independent
|
|
453
|
+
// uniform-in-window anchor (via TimeSoup) and lets the budget loop iterate
|
|
454
|
+
// until the user actually has the target event count.
|
|
455
|
+
//
|
|
456
|
+
// v1.5: when auto-promote marks every event strict (e.g. all events appear in
|
|
457
|
+
// the only funnel), there's nothing for the standalone branch to pick. Bail
|
|
458
|
+
// out cleanly rather than crashing on `pick([])`.
|
|
459
|
+
const hasUsageFunnels = usageFunnels.length > 0;
|
|
460
|
+
const hasStandaloneEvents = weightedEvents.length > 0;
|
|
461
|
+
// Hard ceiling on iterations: defends against pathological configs where
|
|
462
|
+
// every funnel produces 0 surviving events (e.g. ttc > numDays so step1
|
|
463
|
+
// uniform anchor always lands too late). Cap at 2× expected iteration count
|
|
464
|
+
// based on average funnel length (~7 steps) to avoid infinite loops.
|
|
465
|
+
const MAX_ITERATIONS = Math.max(100, numEventsThisUserWillPreform * 2);
|
|
466
|
+
let iterationCount = 0;
|
|
467
|
+
while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled && iterationCount < MAX_ITERATIONS) {
|
|
468
|
+
iterationCount++;
|
|
386
469
|
let newEvents;
|
|
387
|
-
|
|
470
|
+
// v1.5 active-day: pop next picked-day bounds. Pass `latestTime` through
|
|
471
|
+
// featureCtx so makeEvent's TimeSoup confines to that day. Funnel cursor
|
|
472
|
+
// gets re-anchored to the picked day's start (subsequent funnel steps
|
|
473
|
+
// spill within `timeToConvert` hours; this is intentional).
|
|
474
|
+
const dayBounds = dayPlan ? nextDayBounds() : null;
|
|
475
|
+
// Compute step1's `latestTime` so the funnel's relative span fits before
|
|
476
|
+
// FIXED_NOW. Without this, born-late users + long-ttc funnels generate
|
|
477
|
+
// large numbers of `_drop`'d events that consume budget cycles. The
|
|
478
|
+
// safety margin is `timeToConvert * 3600 - 1` seconds (matches v1.5
|
|
479
|
+
// conversion window contract).
|
|
480
|
+
const standaloneFeatureCtx = dayBounds
|
|
481
|
+
? { ...featureCtx, latestTime: dayBounds.latest }
|
|
482
|
+
: featureCtx;
|
|
483
|
+
// Nothing to generate: user converted via firstFunnel only and has no
|
|
484
|
+
// usage funnels and no standalone events. Stop attempting.
|
|
485
|
+
if ((!hasUsageFunnels || !userConverted) && !hasStandaloneEvents) break;
|
|
486
|
+
if (hasUsageFunnels && userConverted) {
|
|
388
487
|
const currentFunnel = chance.pickone(usageFunnels);
|
|
389
|
-
const
|
|
488
|
+
const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
|
|
489
|
+
// Anchor cursor at picked day's start when active-day mode is on,
|
|
490
|
+
// otherwise pass userFirstEventTime (constant). NO cursor accumulation.
|
|
491
|
+
const funnelCursor = dayBounds ? dayBounds.earliest : userFirstEventTime;
|
|
492
|
+
// Constrain funnel step1's TimeSoup latestTime so the full funnel fits in
|
|
493
|
+
// window. Without this, late steps spill past FIXED_NOW and get `_drop`'d.
|
|
494
|
+
//
|
|
495
|
+
// v1.5 final (2026-05-09): `FUNNEL_DEAD_ZONE_CAP_SEC = 0` — funnels can
|
|
496
|
+
// anchor step1 right up to FIXED_NOW. Earlier rounds defended against a
|
|
497
|
+
// cursor-accumulation bug by reserving a `ttc`-sized dead zone at the
|
|
498
|
+
// right edge; round 1 fixed cursor accumulation directly, leaving the
|
|
499
|
+
// dead zone as defense-in-depth. The future-time guard at storage step
|
|
500
|
+
// 14 (per CLAUDE.md "Execution Order") drops any event with `time >
|
|
501
|
+
// FIXED_NOW`, so spillover from late funnel steps is filtered there
|
|
502
|
+
// instead of by anchoring upstream. Removing the dead zone eliminated
|
|
503
|
+
// the last-day cliff for funnel-heavy dungeons WITHOUT re-introducing
|
|
504
|
+
// `futureEvents > 0` — verified across the 194-combo engine-validation
|
|
505
|
+
// sweep (`scripts/sweep-engine.mjs`, `plans/ENGINE-VALIDATION/FIX.md`).
|
|
506
|
+
//
|
|
507
|
+
// Trade-off retained: long-ttc funnels still lose some late steps to
|
|
508
|
+
// `_drop`. Budget loop iterates more to compensate. Catch-all funnel
|
|
509
|
+
// (`ttc=1d`, set in config-validator) is unaffected.
|
|
510
|
+
//
|
|
511
|
+
// Born-late edge case: when `funnelCursor > FN`, the `safeLatest >
|
|
512
|
+
// funnelCursor` check below falls back to `FN` so the user can emit.
|
|
513
|
+
const FUNNEL_DEAD_ZONE_CAP_SEC = 0;
|
|
514
|
+
const deadZoneSec = Math.min(ttcSec, FUNNEL_DEAD_ZONE_CAP_SEC);
|
|
515
|
+
const safeLatest = context.FIXED_NOW - deadZoneSec;
|
|
516
|
+
const funnelLatestTime = dayBounds
|
|
517
|
+
? dayBounds.latest
|
|
518
|
+
: (safeLatest > funnelCursor ? safeLatest : context.FIXED_NOW);
|
|
519
|
+
const funnelEventFeatureCtx = { ...featureCtx, latestTime: funnelLatestTime };
|
|
520
|
+
const [data, converted] = await makeFunnel(context, currentFunnel, user, funnelCursor, profile, userSCD, persona, funnelEventFeatureCtx, usageAttemptMeta);
|
|
521
|
+
// Budget counts raw output (matches pre-fix semantics). For short-ttc
|
|
522
|
+
// funnels (≤1d), almost no events `_drop` so the loop terminates at the
|
|
523
|
+
// expected count. For long-ttc funnels (>1d), some late steps `_drop` —
|
|
524
|
+
// loop iterates more to compensate, world-event `_drop`s still reduce
|
|
525
|
+
// the user's surviving total since they fire on dropped events too.
|
|
390
526
|
numEventsPreformed += data.length;
|
|
391
527
|
newEvents = data;
|
|
392
|
-
if (data.length) {
|
|
393
|
-
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
394
|
-
usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
|
|
395
|
-
}
|
|
396
528
|
} else {
|
|
397
|
-
|
|
529
|
+
// Active-day mode: standalone event uses picked-day start as earliestTime
|
|
530
|
+
// + day-end as latestTime (passed via eventFeatureCtx).
|
|
531
|
+
// `isFirstEvent: false` (8th arg) so TimeSoup distributes the timestamp
|
|
532
|
+
// — passing `true` here would pin every standalone event to the same
|
|
533
|
+
// `earliestTime`, which the now-deleted bunchIntoSessions used to paper
|
|
534
|
+
// over. With bunchIntoSessions removed, TimeSoup is the time source.
|
|
535
|
+
const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
|
|
536
|
+
const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
|
|
398
537
|
numEventsPreformed++;
|
|
399
538
|
newEvents = [data];
|
|
400
539
|
}
|
|
@@ -424,7 +563,7 @@ export async function userLoop(context) {
|
|
|
424
563
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
425
564
|
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
426
565
|
// adjustedCreated and event times now share the same dataset window — no shift.
|
|
427
|
-
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
566
|
+
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance, pickedDayBuckets);
|
|
428
567
|
}
|
|
429
568
|
|
|
430
569
|
// Feature 4: Data quality — duplicates and late-arriving
|
|
@@ -450,18 +589,17 @@ export async function userLoop(context) {
|
|
|
450
589
|
}
|
|
451
590
|
}
|
|
452
591
|
|
|
453
|
-
// Session clustering:
|
|
592
|
+
// Session clustering: assign session IDs based on natural temporal gaps.
|
|
593
|
+
// v1.5: bunchIntoSessions deleted — was a redundant wholesale time-overwrite
|
|
594
|
+
// that scrambled multi-step funnels (round-robin into anchor buckets) and
|
|
595
|
+
// clobbered v1.5 active-day picking. assignSessionIds operates on the original
|
|
596
|
+
// TimeSoup-driven timestamps (Mixpanel-aligned 30-min-gap rule).
|
|
454
597
|
if (hasSessionIds && usersEvents.length > 0) {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
u.bunchIntoSessions(usersEvents, sessionTimeout, {
|
|
461
|
-
earliestTime: userFirstEventTime,
|
|
462
|
-
latestTime: context.FIXED_NOW,
|
|
463
|
-
peaks: soupPeaks, deviation: soupDev, mean: soupMean,
|
|
464
|
-
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
598
|
+
// assignSessionIds requires events sorted ascending by time.
|
|
599
|
+
usersEvents.sort((a, b) => {
|
|
600
|
+
const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
|
|
601
|
+
const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
|
|
602
|
+
return ta - tb;
|
|
465
603
|
});
|
|
466
604
|
u.assignSessionIds(usersEvents, sessionTimeout);
|
|
467
605
|
|
|
@@ -483,6 +621,15 @@ export async function userLoop(context) {
|
|
|
483
621
|
}
|
|
484
622
|
}
|
|
485
623
|
|
|
624
|
+
// v1.5: Touchpoint cap. Sample up to `maxTouchpointsPerUser` (default 10,
|
|
625
|
+
// matching Mixpanel `TOUCHPOINTS_LIMIT`) eligible events from the user's
|
|
626
|
+
// lifetime and stamp UTMs on the sample. Lifetime-distributed sampling
|
|
627
|
+
// preserves realistic touch shape — last-10-window attribution then
|
|
628
|
+
// gives meaningful first/last-touch results.
|
|
629
|
+
if (config.hasCampaigns && usersEvents.length > 0) {
|
|
630
|
+
applyTouchpointCap(usersEvents, config, defaults, chance);
|
|
631
|
+
}
|
|
632
|
+
|
|
486
633
|
// Hook for processing all user events (hooks override everything)
|
|
487
634
|
if (config.hook) {
|
|
488
635
|
// `meta.isPreAuth(event)` predicate bound to this user's auth state.
|
|
@@ -511,14 +658,38 @@ export async function userLoop(context) {
|
|
|
511
658
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
512
659
|
}
|
|
513
660
|
|
|
661
|
+
// v1.5: auto-sort by time after everything hook. Defends against the
|
|
662
|
+
// most common new footgun — hooks that push() cloned events with
|
|
663
|
+
// arbitrary timestamps and break the greedy funnel engine's
|
|
664
|
+
// chronological-order requirement. Opt out with `autoSortAfterEverything: false`.
|
|
665
|
+
if (config.autoSortAfterEverything !== false && usersEvents.length > 1) {
|
|
666
|
+
usersEvents.sort((a, b) => {
|
|
667
|
+
const ta = (a && typeof a.time === 'string') ? Date.parse(a.time) : Number(a && a.time);
|
|
668
|
+
const tb = (b && typeof b.time === 'string') ? Date.parse(b.time) : Number(b && b.time);
|
|
669
|
+
if (!Number.isFinite(ta) && !Number.isFinite(tb)) return 0;
|
|
670
|
+
if (!Number.isFinite(ta)) return 1;
|
|
671
|
+
if (!Number.isFinite(tb)) return -1;
|
|
672
|
+
return ta - tb;
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
|
|
514
676
|
// Defensive guard: drop any events whose timestamp landed past the
|
|
515
677
|
// configured dataset end. Hooks that duplicate events with time offsets
|
|
516
678
|
// (weekend surges, viral spreads) can leak a few past the boundary.
|
|
679
|
+
// v1.5 follow-up (`reccomendations-agent-1.md` Fix #2): surface the drop
|
|
680
|
+
// in verbose mode. Silent dropping made determinism failures hard to debug
|
|
681
|
+
// — events vanished without trace. Per-user log helps the owner see which
|
|
682
|
+
// users + how many events were affected.
|
|
683
|
+
const beforeFutureFilter = usersEvents.length;
|
|
517
684
|
usersEvents = usersEvents.filter(e => {
|
|
518
685
|
if (!e || !e.time) return true;
|
|
519
686
|
const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
|
|
520
687
|
return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
|
|
521
688
|
});
|
|
689
|
+
const droppedFuture = beforeFutureFilter - usersEvents.length;
|
|
690
|
+
if (droppedFuture > 0 && config.verbose) {
|
|
691
|
+
console.warn(`⚠️ Dropped ${droppedFuture} future-dated event(s) for user ${distinct_id}`);
|
|
692
|
+
}
|
|
522
693
|
|
|
523
694
|
// Store all user data
|
|
524
695
|
await userProfilesData.hookPush(profile);
|
|
@@ -552,9 +723,11 @@ export async function userLoop(context) {
|
|
|
552
723
|
if (dataQuality && dataQuality.botUsers > 0) {
|
|
553
724
|
await generateBotUsers(context, dataQuality, storage);
|
|
554
725
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
726
|
+
} finally {
|
|
727
|
+
// Always remove the SIGINT listener — even if userLoop throws or is
|
|
728
|
+
// cancelled. Pre-fix this leaked across test runs and stalled workers.
|
|
729
|
+
process.removeListener('SIGINT', onSigint);
|
|
730
|
+
}
|
|
558
731
|
}
|
|
559
732
|
|
|
560
733
|
|
|
@@ -573,12 +746,261 @@ export function matchConditions(profile, conditions) {
|
|
|
573
746
|
return true;
|
|
574
747
|
}
|
|
575
748
|
|
|
749
|
+
// ── v1.5 Active-Day Plan Helpers ──
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* Build a deterministic per-user "day plan" for active-day mode.
|
|
753
|
+
*
|
|
754
|
+
* Each element of `plan` names the day on which the event-i should land. Events
|
|
755
|
+
* naturally concentrate onto `targetActiveDays` distinct days (drawn from a normal
|
|
756
|
+
* around `avgActiveDaysPerUser`, clamped to `[1, userActiveDays]`).
|
|
757
|
+
*
|
|
758
|
+
* Day picking uses weighted-without-replacement against soup DOW weights so the
|
|
759
|
+
* cohort-level weekly rhythm is preserved. Event distribution across picked days
|
|
760
|
+
* is proportional to those same weights, with a floor of 1 event per picked day.
|
|
761
|
+
*
|
|
762
|
+
* **Return shape (v1.5 follow-up — `reccomendations-agent-1.md` Fix #1):**
|
|
763
|
+
* `{ plan, pickedDayBuckets }` — `plan` is the shuffled per-event day-start
|
|
764
|
+
* unix-seconds array; `pickedDayBuckets` is a `Set<number>` of UTC-day-index
|
|
765
|
+
* buckets (`Math.floor(timestamp_ms / 86400000)`). Downstream consumers like
|
|
766
|
+
* `applyEngagementDecay` use `pickedDayBuckets` to enforce the v1.5
|
|
767
|
+
* distinct-day contract — see Fix #1 below.
|
|
768
|
+
*
|
|
769
|
+
* @param {Object} args
|
|
770
|
+
* @param {import('dayjs').Dayjs} args.adjustedCreated - User's "created" timestamp
|
|
771
|
+
* @param {number} args.fixedBegin - Dataset start (unix seconds)
|
|
772
|
+
* @param {number} args.fixedNow - Dataset end (unix seconds)
|
|
773
|
+
* @param {number} args.avgActiveDaysPerUser
|
|
774
|
+
* @param {number} args.userActiveDays - Capacity (max possible distinct days)
|
|
775
|
+
* @param {number} args.numEvents - Total events to schedule for this user
|
|
776
|
+
* @param {number[]} [args.dowWeights] - 7-element soup DOW weights (Sun..Sat)
|
|
777
|
+
* @param {Object} args.chance - Seeded chance instance
|
|
778
|
+
* @returns {{ plan: number[], pickedDayBuckets: Set<number> } | null} Day plan
|
|
779
|
+
* + bucket set, or null if not applicable (no candidate days, no events, etc.).
|
|
780
|
+
*/
|
|
781
|
+
function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, chance }) {
|
|
782
|
+
if (!Number.isFinite(numEvents) || numEvents <= 0) return null;
|
|
783
|
+
|
|
784
|
+
// Candidate day buckets: UTC days intersecting [max(adjustedCreated, FIXED_BEGIN), FIXED_NOW].
|
|
785
|
+
// Pre-existing users with 'uniform' preExistingSpread have adjustedCreated < FIXED_BEGIN;
|
|
786
|
+
// active-day mode constrains them to in-window days only (matches what Mixpanel sees).
|
|
787
|
+
const userStartUnix = Math.max(
|
|
788
|
+
Math.floor(adjustedCreated.unix()),
|
|
789
|
+
Math.floor(fixedBegin)
|
|
790
|
+
);
|
|
791
|
+
const userEndUnix = Math.floor(fixedNow);
|
|
792
|
+
if (userEndUnix < userStartUnix) return null;
|
|
793
|
+
|
|
794
|
+
const dayMs = 86400;
|
|
795
|
+
const firstDay = Math.floor(userStartUnix / dayMs);
|
|
796
|
+
const lastDay = Math.floor(userEndUnix / dayMs);
|
|
797
|
+
const candidateDays = [];
|
|
798
|
+
for (let d = firstDay; d <= lastDay; d++) {
|
|
799
|
+
candidateDays.push(d * dayMs);
|
|
800
|
+
}
|
|
801
|
+
if (!candidateDays.length) return null;
|
|
802
|
+
|
|
803
|
+
// Weight by soup DOW (Sun=0..Sat=6). Default uniform if no weights configured.
|
|
804
|
+
const weights = candidateDays.map(daySec => {
|
|
805
|
+
if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
|
|
806
|
+
const dow = new Date(daySec * 1000).getUTCDay();
|
|
807
|
+
const w = Number(dowWeights[dow]);
|
|
808
|
+
return Number.isFinite(w) && w > 0 ? w : 0.0001;
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
// Draw target active-day count: normal(mean, sd=mean/3), clamped.
|
|
812
|
+
const meanActive = Math.max(1, Math.min(avgActiveDaysPerUser, candidateDays.length));
|
|
813
|
+
const sd = Math.max(0.5, meanActive / 3);
|
|
814
|
+
let targetActiveDays = Math.round(chance.normal({ mean: meanActive, dev: sd }));
|
|
815
|
+
targetActiveDays = Math.max(1, Math.min(targetActiveDays, candidateDays.length));
|
|
816
|
+
|
|
817
|
+
// Pick which days are active.
|
|
818
|
+
const pickedDays = weightedSampleNoReplacement(candidateDays, weights, targetActiveDays, chance);
|
|
819
|
+
if (!pickedDays.length) return null;
|
|
820
|
+
pickedDays.sort((a, b) => a - b);
|
|
821
|
+
|
|
822
|
+
// Recompute weights aligned to picked days for event distribution.
|
|
823
|
+
const pickedWeights = pickedDays.map(daySec => {
|
|
824
|
+
if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
|
|
825
|
+
const dow = new Date(daySec * 1000).getUTCDay();
|
|
826
|
+
const w = Number(dowWeights[dow]);
|
|
827
|
+
return Number.isFinite(w) && w > 0 ? w : 0.0001;
|
|
828
|
+
});
|
|
829
|
+
const totalWeight = pickedWeights.reduce((s, x) => s + x, 0);
|
|
830
|
+
|
|
831
|
+
// Allocate event counts across picked days. Each picked day gets at least 1
|
|
832
|
+
// event so every day is "used" — distinct-day count actually hits target.
|
|
833
|
+
const k = pickedDays.length;
|
|
834
|
+
let allocations;
|
|
835
|
+
if (numEvents <= k) {
|
|
836
|
+
// Fewer events than picked days: distribute one each, sample which days get them.
|
|
837
|
+
const subPicks = weightedSampleNoReplacement(pickedDays, pickedWeights, numEvents, chance);
|
|
838
|
+
const subSet = new Set(subPicks);
|
|
839
|
+
allocations = pickedDays.map(d => subSet.has(d) ? 1 : 0);
|
|
840
|
+
} else {
|
|
841
|
+
// Floor 1 per picked day, distribute remainder proportionally.
|
|
842
|
+
const remainder = numEvents - k;
|
|
843
|
+
allocations = pickedWeights.map(w => Math.floor((remainder * w) / totalWeight));
|
|
844
|
+
// Add floor of 1 to each
|
|
845
|
+
for (let i = 0; i < k; i++) allocations[i] += 1;
|
|
846
|
+
// Distribute rounding remainder by descending weight order
|
|
847
|
+
let assigned = allocations.reduce((s, x) => s + x, 0);
|
|
848
|
+
let rem = numEvents - assigned;
|
|
849
|
+
const order = pickedDays.map((_, i) => i).sort((a, b) => pickedWeights[b] - pickedWeights[a]);
|
|
850
|
+
for (let i = 0; i < rem; i++) {
|
|
851
|
+
allocations[order[i % order.length]]++;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// Expand into a flat plan and shuffle deterministically.
|
|
856
|
+
const plan = [];
|
|
857
|
+
for (let i = 0; i < k; i++) {
|
|
858
|
+
for (let j = 0; j < allocations[i]; j++) {
|
|
859
|
+
plan.push(pickedDays[i]);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
// v1.5 follow-up (Fix #1): expose the picked-day bucket set so engagement
|
|
863
|
+
// decay can protect the last surviving event per picked day. Buckets use
|
|
864
|
+
// the same `floor(ms / 86400000)` formula that decay applies to event times.
|
|
865
|
+
const pickedDayBuckets = new Set(pickedDays.map(daySec => Math.floor(daySec / 86400)));
|
|
866
|
+
// chance.shuffle uses seeded RNG → deterministic.
|
|
867
|
+
return { plan: chance.shuffle(plan), pickedDayBuckets };
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* v1.5 Touchpoint cap: sample up to `maxTouchpointsPerUser` eligible events from
|
|
872
|
+
* the user's stream and stamp UTMs on the sample. Mirrors Mixpanel's
|
|
873
|
+
* `TOUCHPOINTS_LIMIT = 10` (`backend/libquery/properties_over_time/attributed_value_reader.cpp`).
|
|
874
|
+
*
|
|
875
|
+
* Eligibility:
|
|
876
|
+
* - If any event in `config.events[]` has `isAttributionEvent: true`: candidate
|
|
877
|
+
* pool = events whose name carries that flag.
|
|
878
|
+
* - Else (legacy fallback): candidate pool = ALL events in the user's stream.
|
|
879
|
+
*
|
|
880
|
+
* Sampling:
|
|
881
|
+
* - If `eligible.length <= cap`: stamp all of them.
|
|
882
|
+
* - Else: uniform random sample of size `cap` (seeded `chance.pickset`,
|
|
883
|
+
* deterministic). Sort sample chronologically before stamping so UTMs land
|
|
884
|
+
* in time order.
|
|
885
|
+
*
|
|
886
|
+
* Mutates `events` in place.
|
|
887
|
+
*
|
|
888
|
+
* @param {Object[]} events - User's events (flat shape with .event, .time)
|
|
889
|
+
* @param {Object} config - Validated dungeon config
|
|
890
|
+
* @param {Object} defaults - Context.defaults (provides campaigns())
|
|
891
|
+
* @param {Object} chance - Seeded chance instance
|
|
892
|
+
*/
|
|
893
|
+
function applyTouchpointCap(events, config, defaults, chance) {
|
|
894
|
+
const cap = Number.isFinite(config.maxTouchpointsPerUser)
|
|
895
|
+
? config.maxTouchpointsPerUser
|
|
896
|
+
: 10;
|
|
897
|
+
if (cap <= 0) return;
|
|
898
|
+
|
|
899
|
+
// Build event-name → config map for isAttributionEvent lookup.
|
|
900
|
+
const eventCfgByName = new Map();
|
|
901
|
+
for (const e of (config.events || [])) {
|
|
902
|
+
if (e && e.event) eventCfgByName.set(e.event, e);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// Determine eligible events.
|
|
906
|
+
let eligible;
|
|
907
|
+
if (config.hasAttributionFlags) {
|
|
908
|
+
eligible = events.filter(e => {
|
|
909
|
+
if (!e || !e.event) return false;
|
|
910
|
+
const cfg = eventCfgByName.get(e.event);
|
|
911
|
+
return cfg && cfg.isAttributionEvent === true;
|
|
912
|
+
});
|
|
913
|
+
} else {
|
|
914
|
+
eligible = events.slice();
|
|
915
|
+
}
|
|
916
|
+
if (eligible.length === 0) return;
|
|
917
|
+
|
|
918
|
+
// Sample up to cap (uniform random without replacement, seeded).
|
|
919
|
+
let sample;
|
|
920
|
+
if (eligible.length <= cap) {
|
|
921
|
+
sample = eligible;
|
|
922
|
+
} else {
|
|
923
|
+
sample = chance.pickset(eligible, cap);
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// Sort sample chronologically so UTMs land in time order.
|
|
927
|
+
sample.sort((a, b) => {
|
|
928
|
+
const ta = typeof a.time === 'string' ? Date.parse(a.time) : Number(a.time);
|
|
929
|
+
const tb = typeof b.time === 'string' ? Date.parse(b.time) : Number(b.time);
|
|
930
|
+
return ta - tb;
|
|
931
|
+
});
|
|
932
|
+
|
|
933
|
+
// Stamp UTMs on each sampled event using a campaign template.
|
|
934
|
+
for (const ev of sample) {
|
|
935
|
+
const campaignTemplate = u.pickRandom(defaults.campaigns());
|
|
936
|
+
if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
|
|
937
|
+
for (const [k, v] of Object.entries(campaignTemplate)) {
|
|
938
|
+
ev[k] = u.choose(v);
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Weighted-without-replacement sampler. Samples `count` items from `items` without
|
|
945
|
+
* replacement, with selection probability proportional to `weights`.
|
|
946
|
+
*
|
|
947
|
+
* @param {*[]} items
|
|
948
|
+
* @param {number[]} weights
|
|
949
|
+
* @param {number} count
|
|
950
|
+
* @param {Object} chance - Seeded chance instance
|
|
951
|
+
* @returns {*[]}
|
|
952
|
+
*/
|
|
953
|
+
function weightedSampleNoReplacement(items, weights, count, chance) {
|
|
954
|
+
if (count >= items.length) return items.slice();
|
|
955
|
+
const pool = items.slice();
|
|
956
|
+
const w = weights.slice();
|
|
957
|
+
const result = [];
|
|
958
|
+
for (let i = 0; i < count; i++) {
|
|
959
|
+
const total = w.reduce((s, x) => s + x, 0);
|
|
960
|
+
let chosen;
|
|
961
|
+
if (total <= 0) {
|
|
962
|
+
chosen = chance.integer({ min: 0, max: pool.length - 1 });
|
|
963
|
+
} else {
|
|
964
|
+
let roll = chance.floating({ min: 0, max: total });
|
|
965
|
+
chosen = 0;
|
|
966
|
+
for (let j = 0; j < w.length; j++) {
|
|
967
|
+
roll -= w[j];
|
|
968
|
+
if (roll <= 0) { chosen = j; break; }
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
result.push(pool[chosen]);
|
|
972
|
+
pool.splice(chosen, 1);
|
|
973
|
+
w.splice(chosen, 1);
|
|
974
|
+
}
|
|
975
|
+
return result;
|
|
976
|
+
}
|
|
977
|
+
|
|
576
978
|
// ── Advanced Feature Helper Functions ──
|
|
577
979
|
|
|
578
980
|
/**
|
|
579
|
-
* Feature 3: Apply engagement decay to a user's events
|
|
981
|
+
* Feature 3: Apply engagement decay to a user's events.
|
|
982
|
+
*
|
|
983
|
+
* Filters events probabilistically per the configured decay model
|
|
984
|
+
* (`exponential` / `linear` / `step`). Late events drop more often.
|
|
985
|
+
*
|
|
986
|
+
* **v1.5 follow-up (`reccomendations-agent-1.md` Fix #1):** when
|
|
987
|
+
* `pickedDayBuckets` is provided (active-day mode), the filter NEVER drops the
|
|
988
|
+
* last surviving event on any picked day. Without this, decay can silently
|
|
989
|
+
* undershoot the configured `avgActiveDaysPerUser` by killing all events on
|
|
990
|
+
* sparse late days (e.g., exponential decay with short half-life). The protect-
|
|
991
|
+
* last-event logic preserves the v1.5 distinct-day contract.
|
|
992
|
+
*
|
|
993
|
+
* @param {Object[]} events - User's events (mutated by filter; new array returned)
|
|
994
|
+
* @param {Object} decay - `engagementDecay` config (model/halfLife/floor/etc.)
|
|
995
|
+
* @param {*} userCreated - dayjs object or ISO string of user's creation time
|
|
996
|
+
* @param {Object} context - Engine context (unused; kept for future use)
|
|
997
|
+
* @param {Object} chance - Seeded chance instance
|
|
998
|
+
* @param {Set<number> | null} [pickedDayBuckets=null] - From `buildActiveDayPlan`.
|
|
999
|
+
* Each bucket = `Math.floor(timestamp_ms / 86400000)`. When non-null, the
|
|
1000
|
+
* filter protects the last surviving event on each bucket.
|
|
1001
|
+
* @returns {Object[]} Filtered events array
|
|
580
1002
|
*/
|
|
581
|
-
function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
1003
|
+
function applyEngagementDecay(events, decay, userCreated, context, chance, pickedDayBuckets = null) {
|
|
582
1004
|
if (!events.length) return events;
|
|
583
1005
|
// Perf 2: Use Date.parse instead of dayjs for hot loop
|
|
584
1006
|
const userStartUnix = new Date(userCreated.toISOString ? userCreated.toISOString() : userCreated).getTime() / 1000;
|
|
@@ -587,6 +1009,23 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
587
1009
|
const reactivationChance = decay.reactivationChance || 0;
|
|
588
1010
|
const reactivationMult = decay.reactivationMultiplier || 2.0;
|
|
589
1011
|
|
|
1012
|
+
// v1.5 Fix #1: pre-compute per-bucket counts so we can enforce "at least one
|
|
1013
|
+
// event survives on each picked day". Only counts events that fall within a
|
|
1014
|
+
// picked-day bucket (events spilled to other days don't count toward the
|
|
1015
|
+
// per-bucket survivor budget).
|
|
1016
|
+
const dayMs = 86400000;
|
|
1017
|
+
const dayCounts = new Map(); // bucket → remaining (mutable) event count
|
|
1018
|
+
if (pickedDayBuckets && pickedDayBuckets.size) {
|
|
1019
|
+
for (const ev of events) {
|
|
1020
|
+
const t = new Date(ev.time).getTime();
|
|
1021
|
+
if (!Number.isFinite(t)) continue;
|
|
1022
|
+
const bucket = Math.floor(t / dayMs);
|
|
1023
|
+
if (pickedDayBuckets.has(bucket)) {
|
|
1024
|
+
dayCounts.set(bucket, (dayCounts.get(bucket) || 0) + 1);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|
|
590
1029
|
return events.filter(ev => {
|
|
591
1030
|
const evUnix = new Date(ev.time).getTime() / 1000;
|
|
592
1031
|
const daysSinceBirth = Math.max(0, (evUnix - userStartUnix) / 86400);
|
|
@@ -611,7 +1050,27 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
611
1050
|
retention = Math.min(1.0, retention * reactivationMult);
|
|
612
1051
|
}
|
|
613
1052
|
|
|
614
|
-
|
|
1053
|
+
const keep = chance.bool({ likelihood: retention * 100 });
|
|
1054
|
+
|
|
1055
|
+
// v1.5 Fix #1: protect the last surviving event on each picked day.
|
|
1056
|
+
// Determinism preserved: no extra RNG calls, just a count check.
|
|
1057
|
+
if (pickedDayBuckets && pickedDayBuckets.size) {
|
|
1058
|
+
const bucket = Math.floor(new Date(ev.time).getTime() / dayMs);
|
|
1059
|
+
if (pickedDayBuckets.has(bucket)) {
|
|
1060
|
+
const remaining = dayCounts.get(bucket) || 0;
|
|
1061
|
+
if (!keep) {
|
|
1062
|
+
if (remaining <= 1) {
|
|
1063
|
+
// Last event on this picked day — protect it. Leave dayCounts
|
|
1064
|
+
// at 1 so subsequent events on this bucket can still drop.
|
|
1065
|
+
return true;
|
|
1066
|
+
}
|
|
1067
|
+
dayCounts.set(bucket, remaining - 1);
|
|
1068
|
+
}
|
|
1069
|
+
// If keep===true, no decrement — the survivor still counts.
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
return keep;
|
|
615
1074
|
});
|
|
616
1075
|
}
|
|
617
1076
|
|