@ak--47/dungeon-master 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dungeons/technical/hook-helpers-verify.js +89 -0
- package/dungeons/technical/identity-model-verify.js +47 -0
- package/dungeons/technical/pattern-aggregate-by-bin.js +41 -0
- package/dungeons/technical/pattern-attributed-by-source.js +42 -0
- package/dungeons/technical/pattern-frequency-by-frequency.js +40 -0
- package/dungeons/technical/pattern-funnel-frequency.js +54 -0
- package/dungeons/technical/pattern-ttc-by-segment.js +45 -0
- package/dungeons/vertical/ai-platform.js +45 -52
- package/dungeons/vertical/community.js +11 -8
- package/dungeons/vertical/crypto.js +25 -24
- package/dungeons/vertical/dating.js +56 -48
- package/dungeons/vertical/devtools.js +25 -18
- package/dungeons/vertical/ecommerce.js +42 -38
- package/dungeons/vertical/education.js +24 -9
- package/dungeons/vertical/fintech.js +13 -8
- package/dungeons/vertical/fitness.js +73 -122
- package/dungeons/vertical/food-delivery.js +18 -19
- package/dungeons/vertical/gaming.js +19 -20
- package/dungeons/vertical/healthcare.js +11 -8
- package/dungeons/vertical/insurance-application.js +6 -3
- package/dungeons/vertical/logistics.js +15 -9
- package/dungeons/vertical/marketplace.js +36 -27
- package/dungeons/vertical/media.js +27 -25
- package/dungeons/vertical/real-estate.js +18 -7
- package/dungeons/vertical/sass.js +84 -68
- package/dungeons/vertical/social.js +46 -47
- package/dungeons/vertical/travel.js +8 -5
- package/lib/core/config-validator.js +136 -157
- package/lib/generators/events.js +49 -93
- package/lib/generators/funnels.js +202 -91
- package/lib/hook-helpers/_internal.js +23 -0
- package/lib/hook-helpers/cohort.js +124 -0
- package/lib/hook-helpers/identity.js +56 -0
- package/lib/hook-helpers/index.js +44 -0
- package/lib/hook-helpers/inject.js +99 -0
- package/lib/hook-helpers/mutate.js +151 -0
- package/lib/hook-helpers/timing.js +99 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +38 -0
- package/lib/hook-patterns/attributed-by-source.js +72 -0
- package/lib/hook-patterns/frequency-by-frequency.js +46 -0
- package/lib/hook-patterns/funnel-frequency-breakdown.js +73 -0
- package/lib/hook-patterns/index.js +14 -0
- package/lib/hook-patterns/time-to-convert-by-segment.js +41 -0
- package/lib/orchestrators/user-loop.js +119 -269
- package/lib/utils/utils.js +29 -16
- package/lib/verify/emulate-breakdown.js +281 -0
- package/lib/verify/index.js +12 -0
- package/lib/verify/verify-dungeon.js +61 -0
- package/package.json +6 -4
- package/types.d.ts +397 -211
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
/** @typedef {import('../../types').Context} Context */
|
|
7
7
|
|
|
8
8
|
import dayjs from "dayjs";
|
|
9
|
+
import { randomUUID } from "node:crypto";
|
|
9
10
|
import pLimit from 'p-limit';
|
|
10
11
|
import os from 'os';
|
|
11
12
|
import * as u from "../utils/utils.js";
|
|
@@ -33,6 +34,7 @@ export async function userLoop(context) {
|
|
|
33
34
|
isAnonymous,
|
|
34
35
|
hasAvatar,
|
|
35
36
|
hasAnonIds,
|
|
37
|
+
avgDevicePerUser = 0,
|
|
36
38
|
hasSessionIds,
|
|
37
39
|
hasLocation,
|
|
38
40
|
funnels,
|
|
@@ -48,11 +50,6 @@ export async function userLoop(context) {
|
|
|
48
50
|
worldEvents,
|
|
49
51
|
engagementDecay: globalEngagementDecay,
|
|
50
52
|
dataQuality,
|
|
51
|
-
subscription,
|
|
52
|
-
attribution,
|
|
53
|
-
geo,
|
|
54
|
-
features,
|
|
55
|
-
anomalies
|
|
56
53
|
} = config;
|
|
57
54
|
|
|
58
55
|
const { eventData, userProfilesData, scdTableData } = storage;
|
|
@@ -108,8 +105,9 @@ export async function userLoop(context) {
|
|
|
108
105
|
}
|
|
109
106
|
|
|
110
107
|
const userId = chance.guid();
|
|
111
|
-
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW });
|
|
108
|
+
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
|
|
112
109
|
const { distinct_id, created } = user;
|
|
110
|
+
const userDevicePool = (user.anonymousIds && user.anonymousIds.length) ? user.anonymousIds.slice() : null;
|
|
113
111
|
const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
|
|
114
112
|
|
|
115
113
|
// Feature 1: Assign persona
|
|
@@ -178,41 +176,8 @@ export async function userLoop(context) {
|
|
|
178
176
|
}
|
|
179
177
|
}
|
|
180
178
|
|
|
181
|
-
// Feature 7: Geographic intelligence — assign sticky location
|
|
182
179
|
let userLocation = null;
|
|
183
|
-
|
|
184
|
-
let userTimezoneOffset = 0;
|
|
185
|
-
if (geo && geo.sticky && geo.regions && geo.regions.length > 0) {
|
|
186
|
-
// Assign region using weighted selection
|
|
187
|
-
const regionWeights = geo.regions.map(r => r.weight);
|
|
188
|
-
const totalRegionWeight = regionWeights.reduce((a, b) => a + b, 0);
|
|
189
|
-
let regionRoll = chance.floating({ min: 0, max: totalRegionWeight });
|
|
190
|
-
for (const r of geo.regions) {
|
|
191
|
-
regionRoll -= r.weight;
|
|
192
|
-
if (regionRoll <= 0) { userRegion = r; break; }
|
|
193
|
-
}
|
|
194
|
-
if (!userRegion) userRegion = geo.regions[geo.regions.length - 1];
|
|
195
|
-
userTimezoneOffset = userRegion.timezoneOffset || 0;
|
|
196
|
-
|
|
197
|
-
// Pick a location matching one of this region's countries
|
|
198
|
-
const regionLocations = u.choose(defaults.locationsUsers).filter(
|
|
199
|
-
loc => userRegion.countries.includes(loc.country_code || loc.country)
|
|
200
|
-
);
|
|
201
|
-
if (regionLocations.length > 0) {
|
|
202
|
-
userLocation = u.pickRandom(regionLocations);
|
|
203
|
-
} else {
|
|
204
|
-
userLocation = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
205
|
-
}
|
|
206
|
-
for (const key in userLocation) {
|
|
207
|
-
user[key] = userLocation[key];
|
|
208
|
-
}
|
|
209
|
-
// Inject region properties
|
|
210
|
-
if (userRegion.properties) {
|
|
211
|
-
for (const [k, v] of Object.entries(userRegion.properties)) {
|
|
212
|
-
user[k] = v;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
} else if (hasLocation) {
|
|
180
|
+
if (hasLocation) {
|
|
216
181
|
const location = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
217
182
|
for (const key in location) {
|
|
218
183
|
user[key] = location[key];
|
|
@@ -220,24 +185,6 @@ export async function userLoop(context) {
|
|
|
220
185
|
userLocation = location;
|
|
221
186
|
}
|
|
222
187
|
|
|
223
|
-
// Feature 6: Attribution — assign campaign to users born in dataset
|
|
224
|
-
let userCampaign = null;
|
|
225
|
-
if (attribution && userIsBornInDataset) {
|
|
226
|
-
// adjustedCreated is in the internal FIXED time range
|
|
227
|
-
const birthUnix = adjustedCreated.unix();
|
|
228
|
-
const birthDay = Math.max(0, (birthUnix - context.FIXED_BEGIN) / 86400);
|
|
229
|
-
const isOrganic = chance.bool({ likelihood: (attribution.organicRate || 0.4) * 100 });
|
|
230
|
-
if (!isOrganic) {
|
|
231
|
-
// Find active campaigns at birth day
|
|
232
|
-
const activeCampaigns = attribution.campaigns.filter(c =>
|
|
233
|
-
birthDay >= c.activeDays[0] && birthDay <= c.activeDays[1]
|
|
234
|
-
);
|
|
235
|
-
if (activeCampaigns.length > 0) {
|
|
236
|
-
userCampaign = chance.pickone(activeCampaigns);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
|
|
241
188
|
// Profile creation
|
|
242
189
|
const profile = await makeUserProfile(context, userProps, user);
|
|
243
190
|
|
|
@@ -249,30 +196,12 @@ export async function userLoop(context) {
|
|
|
249
196
|
profile._persona = persona.name;
|
|
250
197
|
}
|
|
251
198
|
|
|
252
|
-
// Feature 6: Add attribution to profile
|
|
253
|
-
if (userCampaign) {
|
|
254
|
-
profile.utm_source = userCampaign.source;
|
|
255
|
-
profile.utm_campaign = userCampaign.name;
|
|
256
|
-
if (userCampaign.medium) profile.utm_medium = userCampaign.medium;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
// Feature 7: Add region to profile
|
|
260
|
-
if (userRegion) {
|
|
261
|
-
profile._region = userRegion.name;
|
|
262
|
-
}
|
|
263
|
-
|
|
264
199
|
// Build feature context for event generation
|
|
265
200
|
const featureCtx = {
|
|
266
201
|
persona,
|
|
267
202
|
userLocation,
|
|
268
203
|
worldEventsTimeline: worldEvents,
|
|
269
|
-
resolvedFeatures: features,
|
|
270
|
-
resolvedAnomalies: anomalies,
|
|
271
204
|
dataQuality,
|
|
272
|
-
geo,
|
|
273
|
-
userCampaign,
|
|
274
|
-
userRegion,
|
|
275
|
-
userTimezoneOffset
|
|
276
205
|
};
|
|
277
206
|
|
|
278
207
|
// Call user hook after profile creation (hooks override persona properties)
|
|
@@ -354,8 +283,9 @@ export async function userLoop(context) {
|
|
|
354
283
|
let usersEvents = [];
|
|
355
284
|
let userConverted = true;
|
|
356
285
|
|
|
357
|
-
// Pre-compute weighted events array for standalone event selection
|
|
358
|
-
|
|
286
|
+
// Pre-compute weighted events array for standalone event selection.
|
|
287
|
+
// Filter out isStrictEvent events — they only appear inside funnels.
|
|
288
|
+
const weightedEvents = config.events.filter(e => !e.isStrictEvent).reduce((acc, event) => {
|
|
359
289
|
const w = Math.max(1, Math.min(Math.floor(event.weight) || 1, 10));
|
|
360
290
|
for (let i = 0; i < w; i++) acc.push(event);
|
|
361
291
|
return acc;
|
|
@@ -369,16 +299,70 @@ export async function userLoop(context) {
|
|
|
369
299
|
}
|
|
370
300
|
}
|
|
371
301
|
|
|
302
|
+
// ── Phase 2 identity tracking ──
|
|
303
|
+
// Pre-existing users are considered already-stitched before the dataset window —
|
|
304
|
+
// `userAuthed` starts true. Born-in-dataset users start anonymous and only flip
|
|
305
|
+
// authed once the stitch (`isAuthEvent` step in their `isFirstFunnel`) actually
|
|
306
|
+
// fires. `userAuthTimeMs` is the unix-millisecond timestamp of that stitch event.
|
|
307
|
+
let userAuthed = !userIsBornInDataset;
|
|
308
|
+
let userAuthTimeMs = null;
|
|
309
|
+
|
|
372
310
|
// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
|
|
373
311
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
374
312
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
313
|
+
let cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
314
|
+
|
|
315
|
+
// Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
|
|
316
|
+
// passes = failedPriors + 1. Validator coerced bounds; default both 0.
|
|
317
|
+
const attemptsCfg = firstFunnel.attempts || null;
|
|
318
|
+
const minA = attemptsCfg ? (attemptsCfg.min || 0) : 0;
|
|
319
|
+
const maxA = attemptsCfg ? (attemptsCfg.max || 0) : 0;
|
|
320
|
+
const failedPriors = (maxA > 0) ? chance.integer({ min: minA, max: maxA }) : 0;
|
|
321
|
+
const totalAttempts = failedPriors + 1;
|
|
322
|
+
let firstAttemptFirstEventTime = null;
|
|
323
|
+
|
|
324
|
+
for (let attemptNum = 1; attemptNum <= totalAttempts; attemptNum++) {
|
|
325
|
+
const isFinal = attemptNum === totalAttempts;
|
|
326
|
+
// On the final attempt, `attempts.conversionRate` (when set) overrides the
|
|
327
|
+
// funnel's normal conversionRate. Clone the funnel object so we don't
|
|
328
|
+
// mutate the shared config.
|
|
329
|
+
const funnelToRun = (isFinal && attemptsCfg && attemptsCfg.conversionRate !== undefined)
|
|
330
|
+
? { ...firstFunnel, conversionRate: attemptsCfg.conversionRate }
|
|
331
|
+
: firstFunnel;
|
|
332
|
+
const attemptMeta = {
|
|
333
|
+
isFirstFunnel: true,
|
|
334
|
+
isBorn: true,
|
|
335
|
+
attemptsConfig: attemptsCfg,
|
|
336
|
+
attemptNumber: attemptNum,
|
|
337
|
+
totalAttempts,
|
|
338
|
+
isFinalAttempt: isFinal,
|
|
339
|
+
truncateBeforeAuth: !isFinal,
|
|
340
|
+
devicePool: userDevicePool,
|
|
341
|
+
};
|
|
342
|
+
const [data, converted, authMs] = await makeFunnel(
|
|
343
|
+
context, funnelToRun, user, cursor, profile, userSCD, persona, featureCtx, attemptMeta
|
|
344
|
+
);
|
|
345
|
+
if (isFinal) userConverted = converted;
|
|
346
|
+
if (data && data.length) {
|
|
347
|
+
if (firstAttemptFirstEventTime === null) {
|
|
348
|
+
firstAttemptFirstEventTime = dayjs(data[0].time).unix();
|
|
349
|
+
}
|
|
350
|
+
// Advance the cursor for the next attempt by a small abandon-and-retry gap.
|
|
351
|
+
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
352
|
+
cursor = lastTime + chance.integer({ min: 60, max: 30 * 60 }); // 1–30 min later
|
|
353
|
+
numEventsPreformed += data.length;
|
|
354
|
+
usersEvents = usersEvents.concat(data);
|
|
355
|
+
}
|
|
356
|
+
if (authMs) {
|
|
357
|
+
// First time we see a stitch wins — should only happen on the final attempt.
|
|
358
|
+
if (userAuthTimeMs === null) userAuthTimeMs = authMs;
|
|
359
|
+
userAuthed = true;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
378
362
|
|
|
379
|
-
userFirstEventTime =
|
|
380
|
-
|
|
381
|
-
|
|
363
|
+
userFirstEventTime = firstAttemptFirstEventTime !== null
|
|
364
|
+
? firstAttemptFirstEventTime
|
|
365
|
+
: adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
382
366
|
} else {
|
|
383
367
|
userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
384
368
|
}
|
|
@@ -387,15 +371,30 @@ export async function userLoop(context) {
|
|
|
387
371
|
let userChurned = false;
|
|
388
372
|
const sessionTimeout = config.sessionTimeout || 30;
|
|
389
373
|
|
|
374
|
+
// Standalone identity stamping mode: pre-existing or post-auth users get both
|
|
375
|
+
// user_id + device_id; born-in-dataset users who never authed (final firstFunnel
|
|
376
|
+
// attempt failed) stay device_only forever per the Phase 2 model.
|
|
377
|
+
const standaloneStamping = userAuthed ? 'both' : 'device_only';
|
|
378
|
+
const standaloneIdentityCtx = (userDevicePool || standaloneStamping !== 'both')
|
|
379
|
+
? { stamping: standaloneStamping, devicePool: userDevicePool }
|
|
380
|
+
: null;
|
|
381
|
+
// Usage funnels for converted users: identity already stitched, just default 'both'.
|
|
382
|
+
const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
|
|
383
|
+
|
|
384
|
+
let usageFunnelCursor = userFirstEventTime;
|
|
390
385
|
while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
|
|
391
386
|
let newEvents;
|
|
392
387
|
if (usageFunnels.length && userConverted) {
|
|
393
388
|
const currentFunnel = chance.pickone(usageFunnels);
|
|
394
|
-
const [data, converted] = await makeFunnel(context, currentFunnel, user,
|
|
389
|
+
const [data, converted] = await makeFunnel(context, currentFunnel, user, usageFunnelCursor, profile, userSCD, persona, featureCtx, usageAttemptMeta);
|
|
395
390
|
numEventsPreformed += data.length;
|
|
396
391
|
newEvents = data;
|
|
392
|
+
if (data.length) {
|
|
393
|
+
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
394
|
+
usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
|
|
395
|
+
}
|
|
397
396
|
} else {
|
|
398
|
-
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds,
|
|
397
|
+
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, true, false, featureCtx, standaloneIdentityCtx);
|
|
399
398
|
numEventsPreformed++;
|
|
400
399
|
newEvents = [data];
|
|
401
400
|
}
|
|
@@ -421,34 +420,13 @@ export async function userLoop(context) {
|
|
|
421
420
|
// Remove events flagged as future timestamps (before dungeon hooks see them)
|
|
422
421
|
usersEvents = usersEvents.filter(e => !e._drop);
|
|
423
422
|
|
|
424
|
-
// Feature 3: Engagement decay — filter behavioral events
|
|
425
|
-
// Subscription events (trial, upgrade, cancel) must not be randomly dropped by decay
|
|
423
|
+
// Feature 3: Engagement decay — filter behavioral events
|
|
426
424
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
427
425
|
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
428
426
|
// adjustedCreated and event times now share the same dataset window — no shift.
|
|
429
427
|
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
430
428
|
}
|
|
431
429
|
|
|
432
|
-
// Feature 5: Subscription lifecycle — inject after decay (exempt from decay filtering)
|
|
433
|
-
if (subscription && userIsBornInDataset) {
|
|
434
|
-
const subEvents = generateSubscriptionEvents(
|
|
435
|
-
subscription, user, persona, adjustedCreated, context, chance
|
|
436
|
-
);
|
|
437
|
-
if (subEvents.length > 0) {
|
|
438
|
-
usersEvents = usersEvents.concat(subEvents);
|
|
439
|
-
// Perf 3: ISO strings sort lexicographically — avoid Date() allocation
|
|
440
|
-
usersEvents.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
441
|
-
}
|
|
442
|
-
// Set current plan on profile
|
|
443
|
-
if (subEvents.length > 0) {
|
|
444
|
-
const lastSubEvent = subEvents[subEvents.length - 1];
|
|
445
|
-
if (lastSubEvent._currentPlan) {
|
|
446
|
-
profile.subscription_plan = lastSubEvent._currentPlan;
|
|
447
|
-
profile.subscription_status = lastSubEvent._subStatus || 'active';
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
|
|
452
430
|
// Feature 4: Data quality — duplicates and late-arriving
|
|
453
431
|
if (dataQuality) {
|
|
454
432
|
if (dataQuality.duplicateRate > 0) {
|
|
@@ -457,9 +435,7 @@ export async function userLoop(context) {
|
|
|
457
435
|
if (chance.bool({ likelihood: dataQuality.duplicateRate * 100 })) {
|
|
458
436
|
const dupe = { ...ev };
|
|
459
437
|
dupe.time = dayjs(ev.time).add(chance.integer({ min: 1, max: 60 }), 'seconds').toISOString();
|
|
460
|
-
|
|
461
|
-
const dupeId = dupe.user_id || dupe.device_id || '';
|
|
462
|
-
dupe.insert_id = u.quickHash(`${dupe.event}-${dupe.time}-${dupeId}-dupe`);
|
|
438
|
+
dupe.insert_id = randomUUID();
|
|
463
439
|
dupes.push(dupe);
|
|
464
440
|
}
|
|
465
441
|
}
|
|
@@ -488,10 +464,39 @@ export async function userLoop(context) {
|
|
|
488
464
|
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
489
465
|
});
|
|
490
466
|
u.assignSessionIds(usersEvents, sessionTimeout);
|
|
467
|
+
|
|
468
|
+
// Phase 2: per-session sticky device. After session_ids exist, deterministically
|
|
469
|
+
// pick one device per session from the user's pool and overwrite each event's
|
|
470
|
+
// device_id so all events in that session share a device. Skip when there's no
|
|
471
|
+
// pool (avgDevicePerUser=0) or only one device (no choice to make).
|
|
472
|
+
if (userDevicePool && userDevicePool.length > 1) {
|
|
473
|
+
const sessionToDevice = new Map();
|
|
474
|
+
for (const ev of usersEvents) {
|
|
475
|
+
if (!ev || !ev.device_id || !ev.session_id) continue;
|
|
476
|
+
let dev = sessionToDevice.get(ev.session_id);
|
|
477
|
+
if (!dev) {
|
|
478
|
+
dev = userDevicePool[Number(u.quickHash(`${distinct_id}:${ev.session_id}`)) % userDevicePool.length];
|
|
479
|
+
sessionToDevice.set(ev.session_id, dev);
|
|
480
|
+
}
|
|
481
|
+
ev.device_id = dev;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
491
484
|
}
|
|
492
485
|
|
|
493
486
|
// Hook for processing all user events (hooks override everything)
|
|
494
487
|
if (config.hook) {
|
|
488
|
+
// `meta.isPreAuth(event)` predicate bound to this user's auth state.
|
|
489
|
+
// - Pre-existing user: authed throughout; never pre-auth.
|
|
490
|
+
// - Born-in-dataset, never converted (userAuthTimeMs===null): all pre-auth.
|
|
491
|
+
// - Born-in-dataset, converted: pre-auth strictly before the stitch event.
|
|
492
|
+
const userAuthTimeMsLocal = userAuthTimeMs;
|
|
493
|
+
const userIsBornLocal = userIsBornInDataset;
|
|
494
|
+
const isPreAuth = (event) => {
|
|
495
|
+
if (!event || !event.time) return false;
|
|
496
|
+
if (userAuthTimeMsLocal === null) return userIsBornLocal;
|
|
497
|
+
const t = typeof event.time === 'string' ? Date.parse(event.time) : Number(event.time);
|
|
498
|
+
return Number.isFinite(t) ? t < userAuthTimeMsLocal : false;
|
|
499
|
+
};
|
|
495
500
|
const newEvents = await config.hook(usersEvents, "everything", {
|
|
496
501
|
profile,
|
|
497
502
|
scd: userSCD,
|
|
@@ -499,7 +504,9 @@ export async function userLoop(context) {
|
|
|
499
504
|
userIsBornInDataset,
|
|
500
505
|
persona,
|
|
501
506
|
datasetStart: context.DATASET_START_SECONDS,
|
|
502
|
-
datasetEnd: context.DATASET_END_SECONDS
|
|
507
|
+
datasetEnd: context.DATASET_END_SECONDS,
|
|
508
|
+
authTime: userAuthTimeMs,
|
|
509
|
+
isPreAuth,
|
|
503
510
|
});
|
|
504
511
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
505
512
|
}
|
|
@@ -546,11 +553,6 @@ export async function userLoop(context) {
|
|
|
546
553
|
await generateBotUsers(context, dataQuality, storage);
|
|
547
554
|
}
|
|
548
555
|
|
|
549
|
-
// Feature 9: Anomaly burst/coordinated injection (after all users)
|
|
550
|
-
if (anomalies) {
|
|
551
|
-
await generateAnomalyBursts(context, anomalies, storage);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
556
|
// Clean up SIGINT handler
|
|
555
557
|
process.removeListener('SIGINT', onSigint);
|
|
556
558
|
}
|
|
@@ -613,121 +615,6 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
613
615
|
});
|
|
614
616
|
}
|
|
615
617
|
|
|
616
|
-
/**
|
|
617
|
-
* Feature 5: Generate subscription lifecycle events for a user
|
|
618
|
-
*/
|
|
619
|
-
function generateSubscriptionEvents(subscription, user, persona, userCreated, context, chance) {
|
|
620
|
-
const { plans, lifecycle, events: eventNames } = subscription;
|
|
621
|
-
const lc = lifecycle;
|
|
622
|
-
const subEvents = [];
|
|
623
|
-
|
|
624
|
-
const defaultPlan = plans.find(p => p.default) || plans[0];
|
|
625
|
-
let currentPlan = defaultPlan;
|
|
626
|
-
let currentStatus = 'active';
|
|
627
|
-
const paidPlans = plans.filter(p => p.price > 0);
|
|
628
|
-
|
|
629
|
-
const userStartUnix = dayjs(userCreated).unix();
|
|
630
|
-
const endUnix = context.FIXED_NOW;
|
|
631
|
-
|
|
632
|
-
// Persona modifiers for subscription behavior
|
|
633
|
-
const personaChurnMod = persona?.churnRate ? (1 + persona.churnRate) : 1.0;
|
|
634
|
-
const personaUpgradeMod = persona?.conversionModifier || 1.0;
|
|
635
|
-
|
|
636
|
-
let currentUnix = userStartUnix;
|
|
637
|
-
const monthSeconds = 30 * 86400;
|
|
638
|
-
|
|
639
|
-
// If the default plan has a trial, start a trial
|
|
640
|
-
const firstPaidPlan = paidPlans[0];
|
|
641
|
-
if (firstPaidPlan && firstPaidPlan.trialDays) {
|
|
642
|
-
const trialStart = currentUnix + chance.integer({ min: 0, max: 86400 });
|
|
643
|
-
if (trialStart < endUnix) {
|
|
644
|
-
subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart, user, firstPaidPlan.name, 'trial'));
|
|
645
|
-
currentUnix = trialStart + (firstPaidPlan.trialDays * 86400);
|
|
646
|
-
|
|
647
|
-
// Trial to paid conversion
|
|
648
|
-
if (chance.bool({ likelihood: lc.trialToPayRate * personaUpgradeMod * 100 })) {
|
|
649
|
-
if (currentUnix < endUnix) {
|
|
650
|
-
subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix, user, firstPaidPlan.name, 'active'));
|
|
651
|
-
currentPlan = firstPaidPlan;
|
|
652
|
-
currentStatus = 'active';
|
|
653
|
-
}
|
|
654
|
-
} else {
|
|
655
|
-
currentStatus = 'expired_trial';
|
|
656
|
-
return subEvents; // didn't convert, no more sub events
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
// Monthly lifecycle loop
|
|
662
|
-
while (currentUnix < endUnix) {
|
|
663
|
-
currentUnix += monthSeconds + chance.integer({ min: -86400, max: 86400 });
|
|
664
|
-
if (currentUnix >= endUnix) break;
|
|
665
|
-
if (currentStatus === 'cancelled') {
|
|
666
|
-
// Win-back check
|
|
667
|
-
if (chance.bool({ likelihood: lc.winBackRate * 100 })) {
|
|
668
|
-
currentUnix += lc.winBackDelay * 86400;
|
|
669
|
-
if (currentUnix >= endUnix) break;
|
|
670
|
-
subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix, user, currentPlan.name, 'active'));
|
|
671
|
-
currentStatus = 'active';
|
|
672
|
-
}
|
|
673
|
-
break;
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
// Payment failure
|
|
677
|
-
if (currentPlan.price > 0 && chance.bool({ likelihood: lc.paymentFailureRate * 100 })) {
|
|
678
|
-
subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix, user, currentPlan.name, 'payment_issue'));
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
// Churn
|
|
682
|
-
if (chance.bool({ likelihood: lc.churnRate * personaChurnMod * 100 })) {
|
|
683
|
-
subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix, user, currentPlan.name, 'cancelled'));
|
|
684
|
-
currentStatus = 'cancelled';
|
|
685
|
-
continue;
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
// Upgrade
|
|
689
|
-
const currentPlanIndex = plans.indexOf(currentPlan);
|
|
690
|
-
if (currentPlanIndex < plans.length - 1 && chance.bool({ likelihood: lc.upgradeRate * personaUpgradeMod * 100 })) {
|
|
691
|
-
const newPlan = plans[currentPlanIndex + 1];
|
|
692
|
-
subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix, user, newPlan.name, 'active', currentPlan.name));
|
|
693
|
-
currentPlan = newPlan;
|
|
694
|
-
continue;
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
// Downgrade
|
|
698
|
-
if (currentPlanIndex > 0 && currentPlan.price > 0 && chance.bool({ likelihood: lc.downgradeRate * 100 })) {
|
|
699
|
-
const newPlan = plans[currentPlanIndex - 1];
|
|
700
|
-
subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix, user, newPlan.name, 'active', currentPlan.name));
|
|
701
|
-
currentPlan = newPlan;
|
|
702
|
-
continue;
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
// Renewal
|
|
706
|
-
if (currentPlan.price > 0) {
|
|
707
|
-
subEvents.push(makeSubEvent(eventNames.renewed, currentUnix, user, currentPlan.name, 'active'));
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
return subEvents;
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
function makeSubEvent(eventName, unixTime, user, planName, status, previousPlan) {
|
|
715
|
-
// Callers already guard `unixTime < endUnix` before invoking, so no clamp here.
|
|
716
|
-
// Previous code clamped against `dayjs()` (wall-clock now) — leaked wall-clock into
|
|
717
|
-
// timestamps and wasn't even the right upper bound.
|
|
718
|
-
const ev = {
|
|
719
|
-
event: eventName,
|
|
720
|
-
time: dayjs.unix(unixTime).toISOString(),
|
|
721
|
-
user_id: user.distinct_id,
|
|
722
|
-
insert_id: u.quickHash(`${eventName}-${unixTime}-${user.distinct_id}`),
|
|
723
|
-
subscription_plan: planName,
|
|
724
|
-
_currentPlan: planName,
|
|
725
|
-
_subStatus: status
|
|
726
|
-
};
|
|
727
|
-
if (previousPlan) ev.previous_plan = previousPlan;
|
|
728
|
-
return ev;
|
|
729
|
-
}
|
|
730
|
-
|
|
731
618
|
/**
|
|
732
619
|
* Feature 4: Generate bot users with repetitive patterns
|
|
733
620
|
*/
|
|
@@ -772,40 +659,3 @@ async function generateBotUsers(context, dataQuality, storage) {
|
|
|
772
659
|
await storage.eventData.hookPush(botEvents, { profile: { distinct_id: botId, is_bot: true } });
|
|
773
660
|
}
|
|
774
661
|
}
|
|
775
|
-
|
|
776
|
-
/**
|
|
777
|
-
* Feature 9: Generate anomaly burst and coordinated events
|
|
778
|
-
*/
|
|
779
|
-
async function generateAnomalyBursts(context, anomalies, storage) {
|
|
780
|
-
const chance = u.getChance();
|
|
781
|
-
|
|
782
|
-
for (const a of anomalies) {
|
|
783
|
-
if (a.type !== 'burst' && a.type !== 'coordinated') continue;
|
|
784
|
-
if (!a._startUnix || !a.count) continue;
|
|
785
|
-
|
|
786
|
-
const burstEvents = [];
|
|
787
|
-
const startUnix = a._startUnix;
|
|
788
|
-
const endUnix = a._endUnix;
|
|
789
|
-
const windowSeconds = endUnix - startUnix;
|
|
790
|
-
|
|
791
|
-
for (let i = 0; i < a.count; i++) {
|
|
792
|
-
const eventTime = startUnix + chance.integer({ min: 0, max: Math.max(1, windowSeconds) });
|
|
793
|
-
if (eventTime > context.FIXED_NOW) continue;
|
|
794
|
-
const userId = a.type === 'coordinated'
|
|
795
|
-
? `anomaly_${chance.guid().slice(0, 8)}`
|
|
796
|
-
: `burst_${chance.integer({ min: 1, max: 100 })}`;
|
|
797
|
-
|
|
798
|
-
const ev = {
|
|
799
|
-
event: a.event,
|
|
800
|
-
time: dayjs.unix(eventTime).toISOString(),
|
|
801
|
-
user_id: userId,
|
|
802
|
-
insert_id: u.quickHash(`${a.event}-${eventTime}-${userId}-${i}`),
|
|
803
|
-
};
|
|
804
|
-
if (a.tag) ev._anomaly = a.tag;
|
|
805
|
-
if (a.properties) Object.assign(ev, a.properties);
|
|
806
|
-
burstEvents.push(ev);
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
await storage.eventData.hookPush(burstEvents, {});
|
|
810
|
-
}
|
|
811
|
-
}
|
package/lib/utils/utils.js
CHANGED
|
@@ -1156,21 +1156,14 @@ CORE
|
|
|
1156
1156
|
*/
|
|
1157
1157
|
|
|
1158
1158
|
//the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
|
|
1159
|
-
function generateUser(user_id, opts
|
|
1159
|
+
function generateUser(user_id, opts) {
|
|
1160
1160
|
const chance = getChance();
|
|
1161
|
-
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix } = opts;
|
|
1162
|
-
//
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
// Sine function for a smoother curve
|
|
1166
|
-
const sineValue = (Math.sin(u * Math.PI * frequency - Math.PI / 2) * amplitude + 1) / 2;
|
|
1167
|
-
|
|
1168
|
-
// Scale the sineValue to the range of days
|
|
1169
|
-
let daysAgoBorn = Math.round(sineValue * (numDays - 1)) + 1;
|
|
1170
|
-
|
|
1171
|
-
// Clamp values to ensure they are within the desired range
|
|
1161
|
+
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix, avgDevicePerUser } = opts;
|
|
1162
|
+
// Birth date placement is overridden by bornRecentBias in user-loop.js;
|
|
1163
|
+
// use a simple uniform distribution as the seed value.
|
|
1164
|
+
let daysAgoBorn = Math.round(chance.random() * (numDays - 1)) + 1;
|
|
1172
1165
|
daysAgoBorn = Math.min(daysAgoBorn, numDays);
|
|
1173
|
-
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix);
|
|
1166
|
+
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix, avgDevicePerUser);
|
|
1174
1167
|
|
|
1175
1168
|
const user = {
|
|
1176
1169
|
distinct_id: user_id,
|
|
@@ -1275,9 +1268,13 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1275
1268
|
* @param {boolean} hasAvatar
|
|
1276
1269
|
* @param {boolean} hasAnonIds
|
|
1277
1270
|
* @param {boolean} hasSessionIds
|
|
1271
|
+
* @param {number} [datasetEndUnix]
|
|
1272
|
+
* @param {number} [avgDevicePerUser] - Whole number ≥ 0. When ≥ 1 (and `hasAnonIds: true`),
|
|
1273
|
+
* sets the per-user device pool size. `1` = exactly one device. `>1` = drawn from
|
|
1274
|
+
* `chance.normal({mean: avgDevicePerUser, dev: avgDevicePerUser/2})`, clamped ≥ 1, integer.
|
|
1278
1275
|
* @return {Person}
|
|
1279
1276
|
*/
|
|
1280
|
-
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false, datasetEndUnix) {
|
|
1277
|
+
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false, datasetEndUnix, avgDevicePerUser) {
|
|
1281
1278
|
const chance = getChance();
|
|
1282
1279
|
//names and photos
|
|
1283
1280
|
const l = chance.letter.bind(chance);
|
|
@@ -1320,9 +1317,25 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
1320
1317
|
|
|
1321
1318
|
if (!hasAvatar) delete user.avatar;
|
|
1322
1319
|
|
|
1323
|
-
//
|
|
1320
|
+
// Device pool ("anonymousIds" — name preserved for backwards compat).
|
|
1321
|
+
// Phase 2 identity model: pool size is governed by `avgDevicePerUser` when set,
|
|
1322
|
+
// else legacy 2–10 random pool size for any dungeon that just sets `hasAnonIds: true`
|
|
1323
|
+
// without `avgDevicePerUser`. (Validator coerces `hasAnonIds: true` to an effective
|
|
1324
|
+
// `avgDevicePerUser >= 1`, so this branch fires whenever there's a device pool to build.)
|
|
1324
1325
|
if (hasAnonIds) {
|
|
1325
|
-
|
|
1326
|
+
let clusterSize;
|
|
1327
|
+
if (typeof avgDevicePerUser === 'number' && avgDevicePerUser >= 1) {
|
|
1328
|
+
if (avgDevicePerUser === 1) {
|
|
1329
|
+
clusterSize = 1;
|
|
1330
|
+
} else {
|
|
1331
|
+
const sd = avgDevicePerUser / 2;
|
|
1332
|
+
const sample = chance.normal({ mean: avgDevicePerUser, dev: sd });
|
|
1333
|
+
clusterSize = Math.max(1, Math.round(sample));
|
|
1334
|
+
}
|
|
1335
|
+
} else {
|
|
1336
|
+
// Legacy fallback (pre-1.4 behavior): random pool of 2–10 devices.
|
|
1337
|
+
clusterSize = integer(2, 10);
|
|
1338
|
+
}
|
|
1326
1339
|
for (let i = 0; i < clusterSize; i++) {
|
|
1327
1340
|
// Use seeded chance, not ak-tools uid() (which uses Math.random).
|
|
1328
1341
|
const anonId = chance.string({ length: 42, pool: 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' });
|