@ak--47/dungeon-master 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +58 -0
  2. package/dungeons/technical/array-of-object-lookup.js +0 -2
  3. package/dungeons/technical/hook-helpers-verify.js +89 -0
  4. package/dungeons/technical/identity-model-verify.js +47 -0
  5. package/dungeons/technical/pattern-aggregate-by-bin.js +41 -0
  6. package/dungeons/technical/pattern-attributed-by-source.js +42 -0
  7. package/dungeons/technical/pattern-frequency-by-frequency.js +40 -0
  8. package/dungeons/technical/pattern-funnel-frequency.js +54 -0
  9. package/dungeons/technical/pattern-ttc-by-segment.js +45 -0
  10. package/dungeons/technical/simple.js +3 -4
  11. package/dungeons/technical/simplest-schema.json +5 -0
  12. package/dungeons/technical/text-generation.js +1 -1
  13. package/dungeons/vertical/ai-platform.js +216 -164
  14. package/dungeons/vertical/community.js +74 -41
  15. package/dungeons/vertical/crypto.js +343 -233
  16. package/dungeons/vertical/dating.js +246 -292
  17. package/dungeons/vertical/devtools.js +179 -70
  18. package/dungeons/vertical/ecommerce.js +275 -122
  19. package/dungeons/vertical/education.js +345 -312
  20. package/dungeons/vertical/fintech.js +445 -314
  21. package/dungeons/vertical/fitness.js +199 -165
  22. package/dungeons/vertical/food-delivery.js +309 -330
  23. package/dungeons/vertical/gaming.js +387 -241
  24. package/dungeons/vertical/healthcare.js +149 -66
  25. package/dungeons/vertical/insurance-application.js +173 -75
  26. package/dungeons/vertical/logistics.js +107 -12
  27. package/dungeons/vertical/marketplace.js +171 -76
  28. package/dungeons/vertical/media.js +259 -402
  29. package/dungeons/vertical/real-estate.js +336 -318
  30. package/dungeons/vertical/sass.js +294 -288
  31. package/dungeons/vertical/social.js +289 -234
  32. package/dungeons/vertical/travel.js +66 -20
  33. package/index.js +17 -17
  34. package/lib/core/config-validator.js +226 -175
  35. package/lib/core/context.js +10 -24
  36. package/lib/core/storage.js +6 -1
  37. package/lib/generators/events.js +57 -104
  38. package/lib/generators/funnels.js +212 -93
  39. package/lib/generators/mirror.js +3 -2
  40. package/lib/generators/product-names.js +1 -1
  41. package/lib/generators/scd.js +2 -1
  42. package/lib/generators/text.js +1 -1
  43. package/lib/hook-helpers/_internal.js +23 -0
  44. package/lib/hook-helpers/cohort.js +124 -0
  45. package/lib/hook-helpers/identity.js +56 -0
  46. package/lib/hook-helpers/index.js +44 -0
  47. package/lib/hook-helpers/inject.js +99 -0
  48. package/lib/hook-helpers/mutate.js +151 -0
  49. package/lib/hook-helpers/timing.js +99 -0
  50. package/lib/hook-patterns/aggregate-per-user-by-bin.js +38 -0
  51. package/lib/hook-patterns/attributed-by-source.js +72 -0
  52. package/lib/hook-patterns/frequency-by-frequency.js +46 -0
  53. package/lib/hook-patterns/funnel-frequency-breakdown.js +73 -0
  54. package/lib/hook-patterns/index.js +14 -0
  55. package/lib/hook-patterns/time-to-convert-by-segment.js +41 -0
  56. package/lib/orchestrators/user-loop.js +143 -293
  57. package/lib/utils/utils.js +97 -52
  58. package/lib/verify/emulate-breakdown.js +281 -0
  59. package/lib/verify/index.js +12 -0
  60. package/lib/verify/verify-dungeon.js +61 -0
  61. package/package.json +6 -4
  62. package/scripts/smoke-test-all.mjs +162 -0
  63. package/scripts/verify-runner.mjs +72 -24
  64. package/types.d.ts +434 -225
  65. package/dungeons/technical/ad-spend-schema.json +0 -128
  66. package/dungeons/technical/anonymous-users-schema.json +0 -92
  67. package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
  68. package/dungeons/technical/experiments-schema.json +0 -203
  69. package/dungeons/technical/foobar-schema.json +0 -362
  70. package/dungeons/technical/group-analytics-schema.json +0 -241
  71. package/dungeons/technical/mirror-strategies-schema.json +0 -84
  72. package/dungeons/technical/nested-objects-schema.json +0 -145
  73. package/dungeons/technical/retention-cadence-schema.json +0 -37
  74. package/dungeons/technical/sanity-schema.json +0 -185
  75. package/dungeons/technical/scale-test-schema.json +0 -70
  76. package/dungeons/technical/scd-schema.json +0 -467
  77. package/dungeons/technical/simple-schema.json +0 -362
  78. package/dungeons/technical/text-generation-schema.json +0 -1062
  79. package/dungeons/vertical/ai-platform-schema.json +0 -617
  80. package/dungeons/vertical/community-schema.json +0 -579
  81. package/dungeons/vertical/crypto-schema.json +0 -546
  82. package/dungeons/vertical/dating-schema.json +0 -401
  83. package/dungeons/vertical/devtools-schema.json +0 -601
  84. package/dungeons/vertical/ecommerce-schema.json +0 -604
  85. package/dungeons/vertical/education-schema.json +0 -5686
  86. package/dungeons/vertical/fintech-schema.json +0 -630
  87. package/dungeons/vertical/fitness-schema.json +0 -530
  88. package/dungeons/vertical/food-delivery-schema.json +0 -36728
  89. package/dungeons/vertical/gaming-schema.json +0 -2703
  90. package/dungeons/vertical/healthcare-schema.json +0 -549
  91. package/dungeons/vertical/insurance-application-schema.json +0 -485
  92. package/dungeons/vertical/logistics-schema.json +0 -574
  93. package/dungeons/vertical/marketplace-schema.json +0 -533
  94. package/dungeons/vertical/media-schema.json +0 -4749
  95. package/dungeons/vertical/real-estate-schema.json +0 -527
  96. package/dungeons/vertical/sass-schema.json +0 -3128
  97. package/dungeons/vertical/social-schema.json +0 -620
  98. package/dungeons/vertical/travel-schema.json +0 -580
@@ -6,6 +6,7 @@
6
6
  /** @typedef {import('../../types').Context} Context */
7
7
 
8
8
  import dayjs from "dayjs";
9
+ import { randomUUID } from "node:crypto";
9
10
  import pLimit from 'p-limit';
10
11
  import os from 'os';
11
12
  import * as u from "../utils/utils.js";
@@ -33,6 +34,7 @@ export async function userLoop(context) {
33
34
  isAnonymous,
34
35
  hasAvatar,
35
36
  hasAnonIds,
37
+ avgDevicePerUser = 0,
36
38
  hasSessionIds,
37
39
  hasLocation,
38
40
  funnels,
@@ -48,11 +50,6 @@ export async function userLoop(context) {
48
50
  worldEvents,
49
51
  engagementDecay: globalEngagementDecay,
50
52
  dataQuality,
51
- subscription,
52
- attribution,
53
- geo,
54
- features,
55
- anomalies
56
53
  } = config;
57
54
 
58
55
  const { eventData, userProfilesData, scdTableData } = storage;
@@ -108,8 +105,9 @@ export async function userLoop(context) {
108
105
  }
109
106
 
110
107
  const userId = chance.guid();
111
- const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
108
+ const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
112
109
  const { distinct_id, created } = user;
110
+ const userDevicePool = (user.anonymousIds && user.anonymousIds.length) ? user.anonymousIds.slice() : null;
113
111
  const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
114
112
 
115
113
  // Feature 1: Assign persona
@@ -128,14 +126,13 @@ export async function userLoop(context) {
128
126
 
129
127
  if (!userIsBornInDataset) delete user.created;
130
128
 
131
- // Calculate time adjustments
132
- const daysShift = context.getDaysShift();
133
-
134
- // Apply recency bias to birth dates for users born in dataset
129
+ // Apply recency bias to birth dates for users born in dataset.
130
+ // `created` (from generateUser → person) is anchored to the dataset end,
131
+ // so it already lives inside [FIXED_BEGIN, FIXED_NOW]. No shift needed.
135
132
  // bornRecentBias: 0 = uniform distribution, 1 = heavily biased toward recent
136
133
  let adjustedCreated;
137
134
  if (userIsBornInDataset) {
138
- let biasedCreated = dayjs(created).subtract(daysShift, 'd');
135
+ let biasedCreated = dayjs(created);
139
136
 
140
137
  if (bornRecentBias !== 0) {
141
138
  // Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
@@ -179,41 +176,8 @@ export async function userLoop(context) {
179
176
  }
180
177
  }
181
178
 
182
- // Feature 7: Geographic intelligence — assign sticky location
183
179
  let userLocation = null;
184
- let userRegion = null;
185
- let userTimezoneOffset = 0;
186
- if (geo && geo.sticky && geo.regions && geo.regions.length > 0) {
187
- // Assign region using weighted selection
188
- const regionWeights = geo.regions.map(r => r.weight);
189
- const totalRegionWeight = regionWeights.reduce((a, b) => a + b, 0);
190
- let regionRoll = chance.floating({ min: 0, max: totalRegionWeight });
191
- for (const r of geo.regions) {
192
- regionRoll -= r.weight;
193
- if (regionRoll <= 0) { userRegion = r; break; }
194
- }
195
- if (!userRegion) userRegion = geo.regions[geo.regions.length - 1];
196
- userTimezoneOffset = userRegion.timezoneOffset || 0;
197
-
198
- // Pick a location matching one of this region's countries
199
- const regionLocations = u.choose(defaults.locationsUsers).filter(
200
- loc => userRegion.countries.includes(loc.country_code || loc.country)
201
- );
202
- if (regionLocations.length > 0) {
203
- userLocation = u.pickRandom(regionLocations);
204
- } else {
205
- userLocation = u.pickRandom(u.choose(defaults.locationsUsers));
206
- }
207
- for (const key in userLocation) {
208
- user[key] = userLocation[key];
209
- }
210
- // Inject region properties
211
- if (userRegion.properties) {
212
- for (const [k, v] of Object.entries(userRegion.properties)) {
213
- user[k] = v;
214
- }
215
- }
216
- } else if (hasLocation) {
180
+ if (hasLocation) {
217
181
  const location = u.pickRandom(u.choose(defaults.locationsUsers));
218
182
  for (const key in location) {
219
183
  user[key] = location[key];
@@ -221,24 +185,6 @@ export async function userLoop(context) {
221
185
  userLocation = location;
222
186
  }
223
187
 
224
- // Feature 6: Attribution — assign campaign to users born in dataset
225
- let userCampaign = null;
226
- if (attribution && userIsBornInDataset) {
227
- // adjustedCreated is in the internal FIXED time range
228
- const birthUnix = adjustedCreated.unix();
229
- const birthDay = Math.max(0, (birthUnix - context.FIXED_BEGIN) / 86400);
230
- const isOrganic = chance.bool({ likelihood: (attribution.organicRate || 0.4) * 100 });
231
- if (!isOrganic) {
232
- // Find active campaigns at birth day
233
- const activeCampaigns = attribution.campaigns.filter(c =>
234
- birthDay >= c.activeDays[0] && birthDay <= c.activeDays[1]
235
- );
236
- if (activeCampaigns.length > 0) {
237
- userCampaign = chance.pickone(activeCampaigns);
238
- }
239
- }
240
- }
241
-
242
188
  // Profile creation
243
189
  const profile = await makeUserProfile(context, userProps, user);
244
190
 
@@ -250,30 +196,12 @@ export async function userLoop(context) {
250
196
  profile._persona = persona.name;
251
197
  }
252
198
 
253
- // Feature 6: Add attribution to profile
254
- if (userCampaign) {
255
- profile.utm_source = userCampaign.source;
256
- profile.utm_campaign = userCampaign.name;
257
- if (userCampaign.medium) profile.utm_medium = userCampaign.medium;
258
- }
259
-
260
- // Feature 7: Add region to profile
261
- if (userRegion) {
262
- profile._region = userRegion.name;
263
- }
264
-
265
199
  // Build feature context for event generation
266
200
  const featureCtx = {
267
201
  persona,
268
202
  userLocation,
269
203
  worldEventsTimeline: worldEvents,
270
- resolvedFeatures: features,
271
- resolvedAnomalies: anomalies,
272
204
  dataQuality,
273
- geo,
274
- userCampaign,
275
- userRegion,
276
- userTimezoneOffset
277
205
  };
278
206
 
279
207
  // Call user hook after profile creation (hooks override persona properties)
@@ -282,7 +210,9 @@ export async function userLoop(context) {
282
210
  user,
283
211
  config,
284
212
  userIsBornInDataset,
285
- persona
213
+ persona,
214
+ datasetStart: context.DATASET_START_SECONDS,
215
+ datasetEnd: context.DATASET_END_SECONDS
286
216
  });
287
217
  }
288
218
 
@@ -304,7 +234,9 @@ export async function userLoop(context) {
304
234
  type: 'user',
305
235
  scd: { [key]: scdProps[key] },
306
236
  config,
307
- allSCDs: userSCD
237
+ allSCDs: userSCD,
238
+ datasetStart: context.DATASET_START_SECONDS,
239
+ datasetEnd: context.DATASET_END_SECONDS
308
240
  });
309
241
  if (Array.isArray(hookResult)) {
310
242
  changes = hookResult;
@@ -351,8 +283,9 @@ export async function userLoop(context) {
351
283
  let usersEvents = [];
352
284
  let userConverted = true;
353
285
 
354
- // Pre-compute weighted events array for standalone event selection
355
- const weightedEvents = config.events.reduce((acc, event) => {
286
+ // Pre-compute weighted events array for standalone event selection.
287
+ // Filter out isStrictEvent events they only appear inside funnels.
288
+ const weightedEvents = config.events.filter(e => !e.isStrictEvent).reduce((acc, event) => {
356
289
  const w = Math.max(1, Math.min(Math.floor(event.weight) || 1, 10));
357
290
  for (let i = 0; i < w; i++) acc.push(event);
358
291
  return acc;
@@ -366,17 +299,70 @@ export async function userLoop(context) {
366
299
  }
367
300
  }
368
301
 
302
+ // ── Phase 2 identity tracking ──
303
+ // Pre-existing users are considered already-stitched before the dataset window —
304
+ // `userAuthed` starts true. Born-in-dataset users start anonymous and only flip
305
+ // authed once the stitch (`isAuthEvent` step in their `isFirstFunnel`) actually
306
+ // fires. `userAuthTimeMs` is the unix-millisecond timestamp of that stitch event.
307
+ let userAuthed = !userIsBornInDataset;
308
+ let userAuthTimeMs = null;
309
+
369
310
  // PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
370
311
  if (firstFunnels.length && userIsBornInDataset) {
371
312
  const firstFunnel = chance.pickone(firstFunnels, user);
372
- const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
373
- const [data, converted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD, persona, featureCtx);
374
- userConverted = converted;
375
-
376
- const timeShift = context.getTimeShift();
377
- userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
378
- numEventsPreformed += data.length;
379
- usersEvents = usersEvents.concat(data);
313
+ let cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
314
+
315
+ // Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
316
+ // passes = failedPriors + 1. Validator coerced bounds; default both 0.
317
+ const attemptsCfg = firstFunnel.attempts || null;
318
+ const minA = attemptsCfg ? (attemptsCfg.min || 0) : 0;
319
+ const maxA = attemptsCfg ? (attemptsCfg.max || 0) : 0;
320
+ const failedPriors = (maxA > 0) ? chance.integer({ min: minA, max: maxA }) : 0;
321
+ const totalAttempts = failedPriors + 1;
322
+ let firstAttemptFirstEventTime = null;
323
+
324
+ for (let attemptNum = 1; attemptNum <= totalAttempts; attemptNum++) {
325
+ const isFinal = attemptNum === totalAttempts;
326
+ // On the final attempt, `attempts.conversionRate` (when set) overrides the
327
+ // funnel's normal conversionRate. Clone the funnel object so we don't
328
+ // mutate the shared config.
329
+ const funnelToRun = (isFinal && attemptsCfg && attemptsCfg.conversionRate !== undefined)
330
+ ? { ...firstFunnel, conversionRate: attemptsCfg.conversionRate }
331
+ : firstFunnel;
332
+ const attemptMeta = {
333
+ isFirstFunnel: true,
334
+ isBorn: true,
335
+ attemptsConfig: attemptsCfg,
336
+ attemptNumber: attemptNum,
337
+ totalAttempts,
338
+ isFinalAttempt: isFinal,
339
+ truncateBeforeAuth: !isFinal,
340
+ devicePool: userDevicePool,
341
+ };
342
+ const [data, converted, authMs] = await makeFunnel(
343
+ context, funnelToRun, user, cursor, profile, userSCD, persona, featureCtx, attemptMeta
344
+ );
345
+ if (isFinal) userConverted = converted;
346
+ if (data && data.length) {
347
+ if (firstAttemptFirstEventTime === null) {
348
+ firstAttemptFirstEventTime = dayjs(data[0].time).unix();
349
+ }
350
+ // Advance the cursor for the next attempt by a small abandon-and-retry gap.
351
+ const lastTime = dayjs(data[data.length - 1].time).unix();
352
+ cursor = lastTime + chance.integer({ min: 60, max: 30 * 60 }); // 1–30 min later
353
+ numEventsPreformed += data.length;
354
+ usersEvents = usersEvents.concat(data);
355
+ }
356
+ if (authMs) {
357
+ // First time we see a stitch wins — should only happen on the final attempt.
358
+ if (userAuthTimeMs === null) userAuthTimeMs = authMs;
359
+ userAuthed = true;
360
+ }
361
+ }
362
+
363
+ userFirstEventTime = firstAttemptFirstEventTime !== null
364
+ ? firstAttemptFirstEventTime
365
+ : adjustedCreated.subtract(noise(), 'seconds').unix();
380
366
  } else {
381
367
  userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
382
368
  }
@@ -385,15 +371,30 @@ export async function userLoop(context) {
385
371
  let userChurned = false;
386
372
  const sessionTimeout = config.sessionTimeout || 30;
387
373
 
374
+ // Standalone identity stamping mode: pre-existing or post-auth users get both
375
+ // user_id + device_id; born-in-dataset users who never authed (final firstFunnel
376
+ // attempt failed) stay device_only forever per the Phase 2 model.
377
+ const standaloneStamping = userAuthed ? 'both' : 'device_only';
378
+ const standaloneIdentityCtx = (userDevicePool || standaloneStamping !== 'both')
379
+ ? { stamping: standaloneStamping, devicePool: userDevicePool }
380
+ : null;
381
+ // Usage funnels for converted users: identity already stitched, just default 'both'.
382
+ const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
383
+
384
+ let usageFunnelCursor = userFirstEventTime;
388
385
  while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
389
386
  let newEvents;
390
387
  if (usageFunnels.length && userConverted) {
391
388
  const currentFunnel = chance.pickone(usageFunnels);
392
- const [data, converted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD, persona, featureCtx);
389
+ const [data, converted] = await makeFunnel(context, currentFunnel, user, usageFunnelCursor, profile, userSCD, persona, featureCtx, usageAttemptMeta);
393
390
  numEventsPreformed += data.length;
394
391
  newEvents = data;
392
+ if (data.length) {
393
+ const lastTime = dayjs(data[data.length - 1].time).unix();
394
+ usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
395
+ }
395
396
  } else {
396
- const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true, false, featureCtx);
397
+ const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, true, false, featureCtx, standaloneIdentityCtx);
397
398
  numEventsPreformed++;
398
399
  newEvents = [data];
399
400
  }
@@ -419,34 +420,11 @@ export async function userLoop(context) {
419
420
  // Remove events flagged as future timestamps (before dungeon hooks see them)
420
421
  usersEvents = usersEvents.filter(e => !e._drop);
421
422
 
422
- // Feature 3: Engagement decay — filter behavioral events BEFORE subscription injection
423
- // Subscription events (trial, upgrade, cancel) must not be randomly dropped by decay
423
+ // Feature 3: Engagement decay — filter behavioral events
424
424
  const userDecay = persona?.engagementDecay || globalEngagementDecay;
425
425
  if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
426
- // adjustedCreated is in FIXED time space, but ev.time is in PRESENT time (shifted).
427
- // Shift adjustedCreated to present so daysSinceBirth reflects within-dataset age.
428
- const adjustedCreatedPresent = adjustedCreated.add(context.TIME_SHIFT_SECONDS, 'seconds');
429
- usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreatedPresent, context, chance);
430
- }
431
-
432
- // Feature 5: Subscription lifecycle — inject after decay (exempt from decay filtering)
433
- if (subscription && userIsBornInDataset) {
434
- const subEvents = generateSubscriptionEvents(
435
- subscription, user, persona, adjustedCreated, context, chance
436
- );
437
- if (subEvents.length > 0) {
438
- usersEvents = usersEvents.concat(subEvents);
439
- // Perf 3: ISO strings sort lexicographically — avoid Date() allocation
440
- usersEvents.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
441
- }
442
- // Set current plan on profile
443
- if (subEvents.length > 0) {
444
- const lastSubEvent = subEvents[subEvents.length - 1];
445
- if (lastSubEvent._currentPlan) {
446
- profile.subscription_plan = lastSubEvent._currentPlan;
447
- profile.subscription_status = lastSubEvent._subStatus || 'active';
448
- }
449
- }
426
+ // adjustedCreated and event times now share the same dataset window no shift.
427
+ usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
450
428
  }
451
429
 
452
430
  // Feature 4: Data quality — duplicates and late-arriving
@@ -457,9 +435,7 @@ export async function userLoop(context) {
457
435
  if (chance.bool({ likelihood: dataQuality.duplicateRate * 100 })) {
458
436
  const dupe = { ...ev };
459
437
  dupe.time = dayjs(ev.time).add(chance.integer({ min: 1, max: 60 }), 'seconds').toISOString();
460
- // Fix 2: Regenerate insert_id so Mixpanel doesn't silently deduplicate
461
- const dupeId = dupe.user_id || dupe.device_id || '';
462
- dupe.insert_id = u.quickHash(`${dupe.event}-${dupe.time}-${dupeId}-dupe`);
438
+ dupe.insert_id = randomUUID();
463
439
  dupes.push(dupe);
464
440
  }
465
441
  }
@@ -485,33 +461,63 @@ export async function userLoop(context) {
485
461
  earliestTime: userFirstEventTime,
486
462
  latestTime: context.FIXED_NOW,
487
463
  peaks: soupPeaks, deviation: soupDev, mean: soupMean,
488
- dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD,
489
- timeShiftSeconds: context.TIME_SHIFT_SECONDS,
490
- maxTime: context.MAX_TIME
464
+ dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
491
465
  });
492
466
  u.assignSessionIds(usersEvents, sessionTimeout);
467
+
468
+ // Phase 2: per-session sticky device. After session_ids exist, deterministically
469
+ // pick one device per session from the user's pool and overwrite each event's
470
+ // device_id so all events in that session share a device. Skip when there's no
471
+ // pool (avgDevicePerUser=0) or only one device (no choice to make).
472
+ if (userDevicePool && userDevicePool.length > 1) {
473
+ const sessionToDevice = new Map();
474
+ for (const ev of usersEvents) {
475
+ if (!ev || !ev.device_id || !ev.session_id) continue;
476
+ let dev = sessionToDevice.get(ev.session_id);
477
+ if (!dev) {
478
+ dev = userDevicePool[Number(u.quickHash(`${distinct_id}:${ev.session_id}`)) % userDevicePool.length];
479
+ sessionToDevice.set(ev.session_id, dev);
480
+ }
481
+ ev.device_id = dev;
482
+ }
483
+ }
493
484
  }
494
485
 
495
486
  // Hook for processing all user events (hooks override everything)
496
487
  if (config.hook) {
488
+ // `meta.isPreAuth(event)` predicate bound to this user's auth state.
489
+ // - Pre-existing user: authed throughout; never pre-auth.
490
+ // - Born-in-dataset, never converted (userAuthTimeMs===null): all pre-auth.
491
+ // - Born-in-dataset, converted: pre-auth strictly before the stitch event.
492
+ const userAuthTimeMsLocal = userAuthTimeMs;
493
+ const userIsBornLocal = userIsBornInDataset;
494
+ const isPreAuth = (event) => {
495
+ if (!event || !event.time) return false;
496
+ if (userAuthTimeMsLocal === null) return userIsBornLocal;
497
+ const t = typeof event.time === 'string' ? Date.parse(event.time) : Number(event.time);
498
+ return Number.isFinite(t) ? t < userAuthTimeMsLocal : false;
499
+ };
497
500
  const newEvents = await config.hook(usersEvents, "everything", {
498
501
  profile,
499
502
  scd: userSCD,
500
503
  config,
501
504
  userIsBornInDataset,
502
- persona
505
+ persona,
506
+ datasetStart: context.DATASET_START_SECONDS,
507
+ datasetEnd: context.DATASET_END_SECONDS,
508
+ authTime: userAuthTimeMs,
509
+ isPreAuth,
503
510
  });
504
511
  if (Array.isArray(newEvents)) usersEvents = newEvents;
505
512
  }
506
513
 
507
- // Defensive guard: drop any events whose timestamp landed past real "now".
508
- // Mixpanel rejects future-dated events, and hooks that duplicate events with
509
- // time offsets (weekend surges, viral spreads) can otherwise leak a few past
510
- // the boundary even after the per-event _drop check in events.js.
514
+ // Defensive guard: drop any events whose timestamp landed past the
515
+ // configured dataset end. Hooks that duplicate events with time offsets
516
+ // (weekend surges, viral spreads) can leak a few past the boundary.
511
517
  usersEvents = usersEvents.filter(e => {
512
518
  if (!e || !e.time) return true;
513
519
  const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
514
- return Number.isFinite(t) ? t <= context.MAX_TIME : true;
520
+ return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
515
521
  });
516
522
 
517
523
  // Store all user data
@@ -547,11 +553,6 @@ export async function userLoop(context) {
547
553
  await generateBotUsers(context, dataQuality, storage);
548
554
  }
549
555
 
550
- // Feature 9: Anomaly burst/coordinated injection (after all users)
551
- if (anomalies) {
552
- await generateAnomalyBursts(context, anomalies, storage);
553
- }
554
-
555
556
  // Clean up SIGINT handler
556
557
  process.removeListener('SIGINT', onSigint);
557
558
  }
@@ -614,119 +615,6 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
614
615
  });
615
616
  }
616
617
 
617
- /**
618
- * Feature 5: Generate subscription lifecycle events for a user
619
- */
620
- function generateSubscriptionEvents(subscription, user, persona, userCreated, context, chance) {
621
- const { plans, lifecycle, events: eventNames } = subscription;
622
- const lc = lifecycle;
623
- const subEvents = [];
624
-
625
- const defaultPlan = plans.find(p => p.default) || plans[0];
626
- let currentPlan = defaultPlan;
627
- let currentStatus = 'active';
628
- const paidPlans = plans.filter(p => p.price > 0);
629
-
630
- const userStartUnix = dayjs(userCreated).unix();
631
- const endUnix = context.MAX_TIME;
632
- const timeShift = context.TIME_SHIFT_SECONDS;
633
-
634
- // Persona modifiers for subscription behavior
635
- const personaChurnMod = persona?.churnRate ? (1 + persona.churnRate) : 1.0;
636
- const personaUpgradeMod = persona?.conversionModifier || 1.0;
637
-
638
- let currentUnix = userStartUnix;
639
- const monthSeconds = 30 * 86400;
640
-
641
- // If the default plan has a trial, start a trial
642
- const firstPaidPlan = paidPlans[0];
643
- if (firstPaidPlan && firstPaidPlan.trialDays) {
644
- const trialStart = currentUnix + chance.integer({ min: 0, max: 86400 });
645
- if (trialStart + timeShift < endUnix) {
646
- subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart + timeShift, user, firstPaidPlan.name, 'trial'));
647
- currentUnix = trialStart + (firstPaidPlan.trialDays * 86400);
648
-
649
- // Trial to paid conversion
650
- if (chance.bool({ likelihood: lc.trialToPayRate * personaUpgradeMod * 100 })) {
651
- if (currentUnix + timeShift < endUnix) {
652
- subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix + timeShift, user, firstPaidPlan.name, 'active'));
653
- currentPlan = firstPaidPlan;
654
- currentStatus = 'active';
655
- }
656
- } else {
657
- currentStatus = 'expired_trial';
658
- return subEvents; // didn't convert, no more sub events
659
- }
660
- }
661
- }
662
-
663
- // Monthly lifecycle loop
664
- while (currentUnix + timeShift < endUnix) {
665
- currentUnix += monthSeconds + chance.integer({ min: -86400, max: 86400 });
666
- if (currentUnix + timeShift >= endUnix) break;
667
- if (currentStatus === 'cancelled') {
668
- // Win-back check
669
- if (chance.bool({ likelihood: lc.winBackRate * 100 })) {
670
- currentUnix += lc.winBackDelay * 86400;
671
- if (currentUnix + timeShift >= endUnix) break;
672
- subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix + timeShift, user, currentPlan.name, 'active'));
673
- currentStatus = 'active';
674
- }
675
- break;
676
- }
677
-
678
- // Payment failure
679
- if (currentPlan.price > 0 && chance.bool({ likelihood: lc.paymentFailureRate * 100 })) {
680
- subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix + timeShift, user, currentPlan.name, 'payment_issue'));
681
- }
682
-
683
- // Churn
684
- if (chance.bool({ likelihood: lc.churnRate * personaChurnMod * 100 })) {
685
- subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix + timeShift, user, currentPlan.name, 'cancelled'));
686
- currentStatus = 'cancelled';
687
- continue;
688
- }
689
-
690
- // Upgrade
691
- const currentPlanIndex = plans.indexOf(currentPlan);
692
- if (currentPlanIndex < plans.length - 1 && chance.bool({ likelihood: lc.upgradeRate * personaUpgradeMod * 100 })) {
693
- const newPlan = plans[currentPlanIndex + 1];
694
- subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
695
- currentPlan = newPlan;
696
- continue;
697
- }
698
-
699
- // Downgrade
700
- if (currentPlanIndex > 0 && currentPlan.price > 0 && chance.bool({ likelihood: lc.downgradeRate * 100 })) {
701
- const newPlan = plans[currentPlanIndex - 1];
702
- subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
703
- currentPlan = newPlan;
704
- continue;
705
- }
706
-
707
- // Renewal
708
- if (currentPlan.price > 0) {
709
- subEvents.push(makeSubEvent(eventNames.renewed, currentUnix + timeShift, user, currentPlan.name, 'active'));
710
- }
711
- }
712
-
713
- return subEvents;
714
- }
715
-
716
- function makeSubEvent(eventName, unixTime, user, planName, status, previousPlan) {
717
- const ev = {
718
- event: eventName,
719
- time: dayjs.unix(Math.min(unixTime, dayjs().unix())).toISOString(),
720
- user_id: user.distinct_id,
721
- insert_id: u.quickHash(`${eventName}-${unixTime}-${user.distinct_id}`),
722
- subscription_plan: planName,
723
- _currentPlan: planName,
724
- _subStatus: status
725
- };
726
- if (previousPlan) ev.previous_plan = previousPlan;
727
- return ev;
728
- }
729
-
730
618
  /**
731
619
  * Feature 4: Generate bot users with repetitive patterns
732
620
  */
@@ -742,14 +630,14 @@ async function generateBotUsers(context, dataQuality, storage) {
742
630
  const botId = `bot_${chance.guid().slice(0, 8)}`;
743
631
  const botEvents = [];
744
632
 
745
- // Bots generate events at machine-like intervals
746
- let currentTime = context.FIXED_BEGIN + context.TIME_SHIFT_SECONDS;
747
- const interval = Math.floor(((context.MAX_TIME - currentTime) / botEventsPerUser));
633
+ // Bots generate events at machine-like intervals across the dataset window
634
+ let currentTime = context.FIXED_BEGIN;
635
+ const interval = Math.floor(((context.FIXED_NOW - currentTime) / botEventsPerUser));
748
636
 
749
637
  for (let e = 0; e < botEventsPerUser; e++) {
750
638
  const chosenEvent = botEventTypes[e % botEventTypes.length];
751
639
  currentTime += interval + chance.integer({ min: 0, max: 10 });
752
- if (currentTime > context.MAX_TIME) break;
640
+ if (currentTime > context.FIXED_NOW) break;
753
641
  botEvents.push({
754
642
  event: chosenEvent.event,
755
643
  time: dayjs.unix(currentTime).toISOString(),
@@ -771,41 +659,3 @@ async function generateBotUsers(context, dataQuality, storage) {
771
659
  await storage.eventData.hookPush(botEvents, { profile: { distinct_id: botId, is_bot: true } });
772
660
  }
773
661
  }
774
-
775
- /**
776
- * Feature 9: Generate anomaly burst and coordinated events
777
- */
778
- async function generateAnomalyBursts(context, anomalies, storage) {
779
- const chance = u.getChance();
780
- const timeShift = context.TIME_SHIFT_SECONDS;
781
-
782
- for (const a of anomalies) {
783
- if (a.type !== 'burst' && a.type !== 'coordinated') continue;
784
- if (!a._startUnix || !a.count) continue;
785
-
786
- const burstEvents = [];
787
- const startUnix = a._startUnix + timeShift;
788
- const endUnix = a._endUnix + timeShift;
789
- const windowSeconds = endUnix - startUnix;
790
-
791
- for (let i = 0; i < a.count; i++) {
792
- const eventTime = startUnix + chance.integer({ min: 0, max: Math.max(1, windowSeconds) });
793
- if (eventTime > context.MAX_TIME) continue;
794
- const userId = a.type === 'coordinated'
795
- ? `anomaly_${chance.guid().slice(0, 8)}`
796
- : `burst_${chance.integer({ min: 1, max: 100 })}`;
797
-
798
- const ev = {
799
- event: a.event,
800
- time: dayjs.unix(eventTime).toISOString(),
801
- user_id: userId,
802
- insert_id: u.quickHash(`${a.event}-${eventTime}-${userId}-${i}`),
803
- };
804
- if (a.tag) ev._anomaly = a.tag;
805
- if (a.properties) Object.assign(ev, a.properties);
806
- burstEvents.push(ev);
807
- }
808
-
809
- await storage.eventData.hookPush(burstEvents, {});
810
- }
811
- }