@ak--47/dungeon-master 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dungeons/technical/array-of-object-lookup.js +0 -2
- package/dungeons/technical/hook-helpers-verify.js +89 -0
- package/dungeons/technical/identity-model-verify.js +47 -0
- package/dungeons/technical/pattern-aggregate-by-bin.js +41 -0
- package/dungeons/technical/pattern-attributed-by-source.js +42 -0
- package/dungeons/technical/pattern-frequency-by-frequency.js +40 -0
- package/dungeons/technical/pattern-funnel-frequency.js +54 -0
- package/dungeons/technical/pattern-ttc-by-segment.js +45 -0
- package/dungeons/technical/simple.js +3 -4
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +216 -164
- package/dungeons/vertical/community.js +74 -41
- package/dungeons/vertical/crypto.js +343 -233
- package/dungeons/vertical/dating.js +246 -292
- package/dungeons/vertical/devtools.js +179 -70
- package/dungeons/vertical/ecommerce.js +275 -122
- package/dungeons/vertical/education.js +345 -312
- package/dungeons/vertical/fintech.js +445 -314
- package/dungeons/vertical/fitness.js +199 -165
- package/dungeons/vertical/food-delivery.js +309 -330
- package/dungeons/vertical/gaming.js +387 -241
- package/dungeons/vertical/healthcare.js +149 -66
- package/dungeons/vertical/insurance-application.js +173 -75
- package/dungeons/vertical/logistics.js +107 -12
- package/dungeons/vertical/marketplace.js +171 -76
- package/dungeons/vertical/media.js +259 -402
- package/dungeons/vertical/real-estate.js +336 -318
- package/dungeons/vertical/sass.js +294 -288
- package/dungeons/vertical/social.js +289 -234
- package/dungeons/vertical/travel.js +66 -20
- package/index.js +17 -17
- package/lib/core/config-validator.js +226 -175
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +57 -104
- package/lib/generators/funnels.js +212 -93
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/hook-helpers/_internal.js +23 -0
- package/lib/hook-helpers/cohort.js +124 -0
- package/lib/hook-helpers/identity.js +56 -0
- package/lib/hook-helpers/index.js +44 -0
- package/lib/hook-helpers/inject.js +99 -0
- package/lib/hook-helpers/mutate.js +151 -0
- package/lib/hook-helpers/timing.js +99 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +38 -0
- package/lib/hook-patterns/attributed-by-source.js +72 -0
- package/lib/hook-patterns/frequency-by-frequency.js +46 -0
- package/lib/hook-patterns/funnel-frequency-breakdown.js +73 -0
- package/lib/hook-patterns/index.js +14 -0
- package/lib/hook-patterns/time-to-convert-by-segment.js +41 -0
- package/lib/orchestrators/user-loop.js +143 -293
- package/lib/utils/utils.js +97 -52
- package/lib/verify/emulate-breakdown.js +281 -0
- package/lib/verify/index.js +12 -0
- package/lib/verify/verify-dungeon.js +61 -0
- package/package.json +6 -4
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +434 -225
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/vertical/ai-platform-schema.json +0 -617
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/crypto-schema.json +0 -546
- package/dungeons/vertical/dating-schema.json +0 -401
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -2703
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/real-estate-schema.json +0 -527
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
/** @typedef {import('../../types').Context} Context */
|
|
7
7
|
|
|
8
8
|
import dayjs from "dayjs";
|
|
9
|
+
import { randomUUID } from "node:crypto";
|
|
9
10
|
import pLimit from 'p-limit';
|
|
10
11
|
import os from 'os';
|
|
11
12
|
import * as u from "../utils/utils.js";
|
|
@@ -33,6 +34,7 @@ export async function userLoop(context) {
|
|
|
33
34
|
isAnonymous,
|
|
34
35
|
hasAvatar,
|
|
35
36
|
hasAnonIds,
|
|
37
|
+
avgDevicePerUser = 0,
|
|
36
38
|
hasSessionIds,
|
|
37
39
|
hasLocation,
|
|
38
40
|
funnels,
|
|
@@ -48,11 +50,6 @@ export async function userLoop(context) {
|
|
|
48
50
|
worldEvents,
|
|
49
51
|
engagementDecay: globalEngagementDecay,
|
|
50
52
|
dataQuality,
|
|
51
|
-
subscription,
|
|
52
|
-
attribution,
|
|
53
|
-
geo,
|
|
54
|
-
features,
|
|
55
|
-
anomalies
|
|
56
53
|
} = config;
|
|
57
54
|
|
|
58
55
|
const { eventData, userProfilesData, scdTableData } = storage;
|
|
@@ -108,8 +105,9 @@ export async function userLoop(context) {
|
|
|
108
105
|
}
|
|
109
106
|
|
|
110
107
|
const userId = chance.guid();
|
|
111
|
-
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
|
|
108
|
+
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
|
|
112
109
|
const { distinct_id, created } = user;
|
|
110
|
+
const userDevicePool = (user.anonymousIds && user.anonymousIds.length) ? user.anonymousIds.slice() : null;
|
|
113
111
|
const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
|
|
114
112
|
|
|
115
113
|
// Feature 1: Assign persona
|
|
@@ -128,14 +126,13 @@ export async function userLoop(context) {
|
|
|
128
126
|
|
|
129
127
|
if (!userIsBornInDataset) delete user.created;
|
|
130
128
|
|
|
131
|
-
//
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
// Apply recency bias to birth dates for users born in dataset
|
|
129
|
+
// Apply recency bias to birth dates for users born in dataset.
|
|
130
|
+
// `created` (from generateUser → person) is anchored to the dataset end,
|
|
131
|
+
// so it already lives inside [FIXED_BEGIN, FIXED_NOW]. No shift needed.
|
|
135
132
|
// bornRecentBias: 0 = uniform distribution, 1 = heavily biased toward recent
|
|
136
133
|
let adjustedCreated;
|
|
137
134
|
if (userIsBornInDataset) {
|
|
138
|
-
let biasedCreated = dayjs(created)
|
|
135
|
+
let biasedCreated = dayjs(created);
|
|
139
136
|
|
|
140
137
|
if (bornRecentBias !== 0) {
|
|
141
138
|
// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
|
|
@@ -179,41 +176,8 @@ export async function userLoop(context) {
|
|
|
179
176
|
}
|
|
180
177
|
}
|
|
181
178
|
|
|
182
|
-
// Feature 7: Geographic intelligence — assign sticky location
|
|
183
179
|
let userLocation = null;
|
|
184
|
-
|
|
185
|
-
let userTimezoneOffset = 0;
|
|
186
|
-
if (geo && geo.sticky && geo.regions && geo.regions.length > 0) {
|
|
187
|
-
// Assign region using weighted selection
|
|
188
|
-
const regionWeights = geo.regions.map(r => r.weight);
|
|
189
|
-
const totalRegionWeight = regionWeights.reduce((a, b) => a + b, 0);
|
|
190
|
-
let regionRoll = chance.floating({ min: 0, max: totalRegionWeight });
|
|
191
|
-
for (const r of geo.regions) {
|
|
192
|
-
regionRoll -= r.weight;
|
|
193
|
-
if (regionRoll <= 0) { userRegion = r; break; }
|
|
194
|
-
}
|
|
195
|
-
if (!userRegion) userRegion = geo.regions[geo.regions.length - 1];
|
|
196
|
-
userTimezoneOffset = userRegion.timezoneOffset || 0;
|
|
197
|
-
|
|
198
|
-
// Pick a location matching one of this region's countries
|
|
199
|
-
const regionLocations = u.choose(defaults.locationsUsers).filter(
|
|
200
|
-
loc => userRegion.countries.includes(loc.country_code || loc.country)
|
|
201
|
-
);
|
|
202
|
-
if (regionLocations.length > 0) {
|
|
203
|
-
userLocation = u.pickRandom(regionLocations);
|
|
204
|
-
} else {
|
|
205
|
-
userLocation = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
206
|
-
}
|
|
207
|
-
for (const key in userLocation) {
|
|
208
|
-
user[key] = userLocation[key];
|
|
209
|
-
}
|
|
210
|
-
// Inject region properties
|
|
211
|
-
if (userRegion.properties) {
|
|
212
|
-
for (const [k, v] of Object.entries(userRegion.properties)) {
|
|
213
|
-
user[k] = v;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
} else if (hasLocation) {
|
|
180
|
+
if (hasLocation) {
|
|
217
181
|
const location = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
218
182
|
for (const key in location) {
|
|
219
183
|
user[key] = location[key];
|
|
@@ -221,24 +185,6 @@ export async function userLoop(context) {
|
|
|
221
185
|
userLocation = location;
|
|
222
186
|
}
|
|
223
187
|
|
|
224
|
-
// Feature 6: Attribution — assign campaign to users born in dataset
|
|
225
|
-
let userCampaign = null;
|
|
226
|
-
if (attribution && userIsBornInDataset) {
|
|
227
|
-
// adjustedCreated is in the internal FIXED time range
|
|
228
|
-
const birthUnix = adjustedCreated.unix();
|
|
229
|
-
const birthDay = Math.max(0, (birthUnix - context.FIXED_BEGIN) / 86400);
|
|
230
|
-
const isOrganic = chance.bool({ likelihood: (attribution.organicRate || 0.4) * 100 });
|
|
231
|
-
if (!isOrganic) {
|
|
232
|
-
// Find active campaigns at birth day
|
|
233
|
-
const activeCampaigns = attribution.campaigns.filter(c =>
|
|
234
|
-
birthDay >= c.activeDays[0] && birthDay <= c.activeDays[1]
|
|
235
|
-
);
|
|
236
|
-
if (activeCampaigns.length > 0) {
|
|
237
|
-
userCampaign = chance.pickone(activeCampaigns);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
188
|
// Profile creation
|
|
243
189
|
const profile = await makeUserProfile(context, userProps, user);
|
|
244
190
|
|
|
@@ -250,30 +196,12 @@ export async function userLoop(context) {
|
|
|
250
196
|
profile._persona = persona.name;
|
|
251
197
|
}
|
|
252
198
|
|
|
253
|
-
// Feature 6: Add attribution to profile
|
|
254
|
-
if (userCampaign) {
|
|
255
|
-
profile.utm_source = userCampaign.source;
|
|
256
|
-
profile.utm_campaign = userCampaign.name;
|
|
257
|
-
if (userCampaign.medium) profile.utm_medium = userCampaign.medium;
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
// Feature 7: Add region to profile
|
|
261
|
-
if (userRegion) {
|
|
262
|
-
profile._region = userRegion.name;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
199
|
// Build feature context for event generation
|
|
266
200
|
const featureCtx = {
|
|
267
201
|
persona,
|
|
268
202
|
userLocation,
|
|
269
203
|
worldEventsTimeline: worldEvents,
|
|
270
|
-
resolvedFeatures: features,
|
|
271
|
-
resolvedAnomalies: anomalies,
|
|
272
204
|
dataQuality,
|
|
273
|
-
geo,
|
|
274
|
-
userCampaign,
|
|
275
|
-
userRegion,
|
|
276
|
-
userTimezoneOffset
|
|
277
205
|
};
|
|
278
206
|
|
|
279
207
|
// Call user hook after profile creation (hooks override persona properties)
|
|
@@ -282,7 +210,9 @@ export async function userLoop(context) {
|
|
|
282
210
|
user,
|
|
283
211
|
config,
|
|
284
212
|
userIsBornInDataset,
|
|
285
|
-
persona
|
|
213
|
+
persona,
|
|
214
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
215
|
+
datasetEnd: context.DATASET_END_SECONDS
|
|
286
216
|
});
|
|
287
217
|
}
|
|
288
218
|
|
|
@@ -304,7 +234,9 @@ export async function userLoop(context) {
|
|
|
304
234
|
type: 'user',
|
|
305
235
|
scd: { [key]: scdProps[key] },
|
|
306
236
|
config,
|
|
307
|
-
allSCDs: userSCD
|
|
237
|
+
allSCDs: userSCD,
|
|
238
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
239
|
+
datasetEnd: context.DATASET_END_SECONDS
|
|
308
240
|
});
|
|
309
241
|
if (Array.isArray(hookResult)) {
|
|
310
242
|
changes = hookResult;
|
|
@@ -351,8 +283,9 @@ export async function userLoop(context) {
|
|
|
351
283
|
let usersEvents = [];
|
|
352
284
|
let userConverted = true;
|
|
353
285
|
|
|
354
|
-
// Pre-compute weighted events array for standalone event selection
|
|
355
|
-
|
|
286
|
+
// Pre-compute weighted events array for standalone event selection.
|
|
287
|
+
// Filter out isStrictEvent events — they only appear inside funnels.
|
|
288
|
+
const weightedEvents = config.events.filter(e => !e.isStrictEvent).reduce((acc, event) => {
|
|
356
289
|
const w = Math.max(1, Math.min(Math.floor(event.weight) || 1, 10));
|
|
357
290
|
for (let i = 0; i < w; i++) acc.push(event);
|
|
358
291
|
return acc;
|
|
@@ -366,17 +299,70 @@ export async function userLoop(context) {
|
|
|
366
299
|
}
|
|
367
300
|
}
|
|
368
301
|
|
|
302
|
+
// ── Phase 2 identity tracking ──
|
|
303
|
+
// Pre-existing users are considered already-stitched before the dataset window —
|
|
304
|
+
// `userAuthed` starts true. Born-in-dataset users start anonymous and only flip
|
|
305
|
+
// authed once the stitch (`isAuthEvent` step in their `isFirstFunnel`) actually
|
|
306
|
+
// fires. `userAuthTimeMs` is the unix-millisecond timestamp of that stitch event.
|
|
307
|
+
let userAuthed = !userIsBornInDataset;
|
|
308
|
+
let userAuthTimeMs = null;
|
|
309
|
+
|
|
369
310
|
// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
|
|
370
311
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
371
312
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
313
|
+
let cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
314
|
+
|
|
315
|
+
// Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
|
|
316
|
+
// passes = failedPriors + 1. Validator coerced bounds; default both 0.
|
|
317
|
+
const attemptsCfg = firstFunnel.attempts || null;
|
|
318
|
+
const minA = attemptsCfg ? (attemptsCfg.min || 0) : 0;
|
|
319
|
+
const maxA = attemptsCfg ? (attemptsCfg.max || 0) : 0;
|
|
320
|
+
const failedPriors = (maxA > 0) ? chance.integer({ min: minA, max: maxA }) : 0;
|
|
321
|
+
const totalAttempts = failedPriors + 1;
|
|
322
|
+
let firstAttemptFirstEventTime = null;
|
|
323
|
+
|
|
324
|
+
for (let attemptNum = 1; attemptNum <= totalAttempts; attemptNum++) {
|
|
325
|
+
const isFinal = attemptNum === totalAttempts;
|
|
326
|
+
// On the final attempt, `attempts.conversionRate` (when set) overrides the
|
|
327
|
+
// funnel's normal conversionRate. Clone the funnel object so we don't
|
|
328
|
+
// mutate the shared config.
|
|
329
|
+
const funnelToRun = (isFinal && attemptsCfg && attemptsCfg.conversionRate !== undefined)
|
|
330
|
+
? { ...firstFunnel, conversionRate: attemptsCfg.conversionRate }
|
|
331
|
+
: firstFunnel;
|
|
332
|
+
const attemptMeta = {
|
|
333
|
+
isFirstFunnel: true,
|
|
334
|
+
isBorn: true,
|
|
335
|
+
attemptsConfig: attemptsCfg,
|
|
336
|
+
attemptNumber: attemptNum,
|
|
337
|
+
totalAttempts,
|
|
338
|
+
isFinalAttempt: isFinal,
|
|
339
|
+
truncateBeforeAuth: !isFinal,
|
|
340
|
+
devicePool: userDevicePool,
|
|
341
|
+
};
|
|
342
|
+
const [data, converted, authMs] = await makeFunnel(
|
|
343
|
+
context, funnelToRun, user, cursor, profile, userSCD, persona, featureCtx, attemptMeta
|
|
344
|
+
);
|
|
345
|
+
if (isFinal) userConverted = converted;
|
|
346
|
+
if (data && data.length) {
|
|
347
|
+
if (firstAttemptFirstEventTime === null) {
|
|
348
|
+
firstAttemptFirstEventTime = dayjs(data[0].time).unix();
|
|
349
|
+
}
|
|
350
|
+
// Advance the cursor for the next attempt by a small abandon-and-retry gap.
|
|
351
|
+
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
352
|
+
cursor = lastTime + chance.integer({ min: 60, max: 30 * 60 }); // 1–30 min later
|
|
353
|
+
numEventsPreformed += data.length;
|
|
354
|
+
usersEvents = usersEvents.concat(data);
|
|
355
|
+
}
|
|
356
|
+
if (authMs) {
|
|
357
|
+
// First time we see a stitch wins — should only happen on the final attempt.
|
|
358
|
+
if (userAuthTimeMs === null) userAuthTimeMs = authMs;
|
|
359
|
+
userAuthed = true;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
userFirstEventTime = firstAttemptFirstEventTime !== null
|
|
364
|
+
? firstAttemptFirstEventTime
|
|
365
|
+
: adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
380
366
|
} else {
|
|
381
367
|
userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
382
368
|
}
|
|
@@ -385,15 +371,30 @@ export async function userLoop(context) {
|
|
|
385
371
|
let userChurned = false;
|
|
386
372
|
const sessionTimeout = config.sessionTimeout || 30;
|
|
387
373
|
|
|
374
|
+
// Standalone identity stamping mode: pre-existing or post-auth users get both
|
|
375
|
+
// user_id + device_id; born-in-dataset users who never authed (final firstFunnel
|
|
376
|
+
// attempt failed) stay device_only forever per the Phase 2 model.
|
|
377
|
+
const standaloneStamping = userAuthed ? 'both' : 'device_only';
|
|
378
|
+
const standaloneIdentityCtx = (userDevicePool || standaloneStamping !== 'both')
|
|
379
|
+
? { stamping: standaloneStamping, devicePool: userDevicePool }
|
|
380
|
+
: null;
|
|
381
|
+
// Usage funnels for converted users: identity already stitched, just default 'both'.
|
|
382
|
+
const usageAttemptMeta = { isFirstFunnel: false, isBorn: userIsBornInDataset, devicePool: userDevicePool };
|
|
383
|
+
|
|
384
|
+
let usageFunnelCursor = userFirstEventTime;
|
|
388
385
|
while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
|
|
389
386
|
let newEvents;
|
|
390
387
|
if (usageFunnels.length && userConverted) {
|
|
391
388
|
const currentFunnel = chance.pickone(usageFunnels);
|
|
392
|
-
const [data, converted] = await makeFunnel(context, currentFunnel, user,
|
|
389
|
+
const [data, converted] = await makeFunnel(context, currentFunnel, user, usageFunnelCursor, profile, userSCD, persona, featureCtx, usageAttemptMeta);
|
|
393
390
|
numEventsPreformed += data.length;
|
|
394
391
|
newEvents = data;
|
|
392
|
+
if (data.length) {
|
|
393
|
+
const lastTime = dayjs(data[data.length - 1].time).unix();
|
|
394
|
+
usageFunnelCursor = lastTime + chance.integer({ min: 60, max: 30 * 60 });
|
|
395
|
+
}
|
|
395
396
|
} else {
|
|
396
|
-
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds,
|
|
397
|
+
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, true, false, featureCtx, standaloneIdentityCtx);
|
|
397
398
|
numEventsPreformed++;
|
|
398
399
|
newEvents = [data];
|
|
399
400
|
}
|
|
@@ -419,34 +420,11 @@ export async function userLoop(context) {
|
|
|
419
420
|
// Remove events flagged as future timestamps (before dungeon hooks see them)
|
|
420
421
|
usersEvents = usersEvents.filter(e => !e._drop);
|
|
421
422
|
|
|
422
|
-
// Feature 3: Engagement decay — filter behavioral events
|
|
423
|
-
// Subscription events (trial, upgrade, cancel) must not be randomly dropped by decay
|
|
423
|
+
// Feature 3: Engagement decay — filter behavioral events
|
|
424
424
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
425
425
|
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
426
|
-
// adjustedCreated
|
|
427
|
-
|
|
428
|
-
const adjustedCreatedPresent = adjustedCreated.add(context.TIME_SHIFT_SECONDS, 'seconds');
|
|
429
|
-
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreatedPresent, context, chance);
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
// Feature 5: Subscription lifecycle — inject after decay (exempt from decay filtering)
|
|
433
|
-
if (subscription && userIsBornInDataset) {
|
|
434
|
-
const subEvents = generateSubscriptionEvents(
|
|
435
|
-
subscription, user, persona, adjustedCreated, context, chance
|
|
436
|
-
);
|
|
437
|
-
if (subEvents.length > 0) {
|
|
438
|
-
usersEvents = usersEvents.concat(subEvents);
|
|
439
|
-
// Perf 3: ISO strings sort lexicographically — avoid Date() allocation
|
|
440
|
-
usersEvents.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
441
|
-
}
|
|
442
|
-
// Set current plan on profile
|
|
443
|
-
if (subEvents.length > 0) {
|
|
444
|
-
const lastSubEvent = subEvents[subEvents.length - 1];
|
|
445
|
-
if (lastSubEvent._currentPlan) {
|
|
446
|
-
profile.subscription_plan = lastSubEvent._currentPlan;
|
|
447
|
-
profile.subscription_status = lastSubEvent._subStatus || 'active';
|
|
448
|
-
}
|
|
449
|
-
}
|
|
426
|
+
// adjustedCreated and event times now share the same dataset window — no shift.
|
|
427
|
+
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
450
428
|
}
|
|
451
429
|
|
|
452
430
|
// Feature 4: Data quality — duplicates and late-arriving
|
|
@@ -457,9 +435,7 @@ export async function userLoop(context) {
|
|
|
457
435
|
if (chance.bool({ likelihood: dataQuality.duplicateRate * 100 })) {
|
|
458
436
|
const dupe = { ...ev };
|
|
459
437
|
dupe.time = dayjs(ev.time).add(chance.integer({ min: 1, max: 60 }), 'seconds').toISOString();
|
|
460
|
-
|
|
461
|
-
const dupeId = dupe.user_id || dupe.device_id || '';
|
|
462
|
-
dupe.insert_id = u.quickHash(`${dupe.event}-${dupe.time}-${dupeId}-dupe`);
|
|
438
|
+
dupe.insert_id = randomUUID();
|
|
463
439
|
dupes.push(dupe);
|
|
464
440
|
}
|
|
465
441
|
}
|
|
@@ -485,33 +461,63 @@ export async function userLoop(context) {
|
|
|
485
461
|
earliestTime: userFirstEventTime,
|
|
486
462
|
latestTime: context.FIXED_NOW,
|
|
487
463
|
peaks: soupPeaks, deviation: soupDev, mean: soupMean,
|
|
488
|
-
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
489
|
-
timeShiftSeconds: context.TIME_SHIFT_SECONDS,
|
|
490
|
-
maxTime: context.MAX_TIME
|
|
464
|
+
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
491
465
|
});
|
|
492
466
|
u.assignSessionIds(usersEvents, sessionTimeout);
|
|
467
|
+
|
|
468
|
+
// Phase 2: per-session sticky device. After session_ids exist, deterministically
|
|
469
|
+
// pick one device per session from the user's pool and overwrite each event's
|
|
470
|
+
// device_id so all events in that session share a device. Skip when there's no
|
|
471
|
+
// pool (avgDevicePerUser=0) or only one device (no choice to make).
|
|
472
|
+
if (userDevicePool && userDevicePool.length > 1) {
|
|
473
|
+
const sessionToDevice = new Map();
|
|
474
|
+
for (const ev of usersEvents) {
|
|
475
|
+
if (!ev || !ev.device_id || !ev.session_id) continue;
|
|
476
|
+
let dev = sessionToDevice.get(ev.session_id);
|
|
477
|
+
if (!dev) {
|
|
478
|
+
dev = userDevicePool[Number(u.quickHash(`${distinct_id}:${ev.session_id}`)) % userDevicePool.length];
|
|
479
|
+
sessionToDevice.set(ev.session_id, dev);
|
|
480
|
+
}
|
|
481
|
+
ev.device_id = dev;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
493
484
|
}
|
|
494
485
|
|
|
495
486
|
// Hook for processing all user events (hooks override everything)
|
|
496
487
|
if (config.hook) {
|
|
488
|
+
// `meta.isPreAuth(event)` predicate bound to this user's auth state.
|
|
489
|
+
// - Pre-existing user: authed throughout; never pre-auth.
|
|
490
|
+
// - Born-in-dataset, never converted (userAuthTimeMs===null): all pre-auth.
|
|
491
|
+
// - Born-in-dataset, converted: pre-auth strictly before the stitch event.
|
|
492
|
+
const userAuthTimeMsLocal = userAuthTimeMs;
|
|
493
|
+
const userIsBornLocal = userIsBornInDataset;
|
|
494
|
+
const isPreAuth = (event) => {
|
|
495
|
+
if (!event || !event.time) return false;
|
|
496
|
+
if (userAuthTimeMsLocal === null) return userIsBornLocal;
|
|
497
|
+
const t = typeof event.time === 'string' ? Date.parse(event.time) : Number(event.time);
|
|
498
|
+
return Number.isFinite(t) ? t < userAuthTimeMsLocal : false;
|
|
499
|
+
};
|
|
497
500
|
const newEvents = await config.hook(usersEvents, "everything", {
|
|
498
501
|
profile,
|
|
499
502
|
scd: userSCD,
|
|
500
503
|
config,
|
|
501
504
|
userIsBornInDataset,
|
|
502
|
-
persona
|
|
505
|
+
persona,
|
|
506
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
507
|
+
datasetEnd: context.DATASET_END_SECONDS,
|
|
508
|
+
authTime: userAuthTimeMs,
|
|
509
|
+
isPreAuth,
|
|
503
510
|
});
|
|
504
511
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
505
512
|
}
|
|
506
513
|
|
|
507
|
-
// Defensive guard: drop any events whose timestamp landed past
|
|
508
|
-
//
|
|
509
|
-
//
|
|
510
|
-
// the boundary even after the per-event _drop check in events.js.
|
|
514
|
+
// Defensive guard: drop any events whose timestamp landed past the
|
|
515
|
+
// configured dataset end. Hooks that duplicate events with time offsets
|
|
516
|
+
// (weekend surges, viral spreads) can leak a few past the boundary.
|
|
511
517
|
usersEvents = usersEvents.filter(e => {
|
|
512
518
|
if (!e || !e.time) return true;
|
|
513
519
|
const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
|
|
514
|
-
return Number.isFinite(t) ? t <= context.
|
|
520
|
+
return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
|
|
515
521
|
});
|
|
516
522
|
|
|
517
523
|
// Store all user data
|
|
@@ -547,11 +553,6 @@ export async function userLoop(context) {
|
|
|
547
553
|
await generateBotUsers(context, dataQuality, storage);
|
|
548
554
|
}
|
|
549
555
|
|
|
550
|
-
// Feature 9: Anomaly burst/coordinated injection (after all users)
|
|
551
|
-
if (anomalies) {
|
|
552
|
-
await generateAnomalyBursts(context, anomalies, storage);
|
|
553
|
-
}
|
|
554
|
-
|
|
555
556
|
// Clean up SIGINT handler
|
|
556
557
|
process.removeListener('SIGINT', onSigint);
|
|
557
558
|
}
|
|
@@ -614,119 +615,6 @@ function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
|
614
615
|
});
|
|
615
616
|
}
|
|
616
617
|
|
|
617
|
-
/**
|
|
618
|
-
* Feature 5: Generate subscription lifecycle events for a user
|
|
619
|
-
*/
|
|
620
|
-
function generateSubscriptionEvents(subscription, user, persona, userCreated, context, chance) {
|
|
621
|
-
const { plans, lifecycle, events: eventNames } = subscription;
|
|
622
|
-
const lc = lifecycle;
|
|
623
|
-
const subEvents = [];
|
|
624
|
-
|
|
625
|
-
const defaultPlan = plans.find(p => p.default) || plans[0];
|
|
626
|
-
let currentPlan = defaultPlan;
|
|
627
|
-
let currentStatus = 'active';
|
|
628
|
-
const paidPlans = plans.filter(p => p.price > 0);
|
|
629
|
-
|
|
630
|
-
const userStartUnix = dayjs(userCreated).unix();
|
|
631
|
-
const endUnix = context.MAX_TIME;
|
|
632
|
-
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
633
|
-
|
|
634
|
-
// Persona modifiers for subscription behavior
|
|
635
|
-
const personaChurnMod = persona?.churnRate ? (1 + persona.churnRate) : 1.0;
|
|
636
|
-
const personaUpgradeMod = persona?.conversionModifier || 1.0;
|
|
637
|
-
|
|
638
|
-
let currentUnix = userStartUnix;
|
|
639
|
-
const monthSeconds = 30 * 86400;
|
|
640
|
-
|
|
641
|
-
// If the default plan has a trial, start a trial
|
|
642
|
-
const firstPaidPlan = paidPlans[0];
|
|
643
|
-
if (firstPaidPlan && firstPaidPlan.trialDays) {
|
|
644
|
-
const trialStart = currentUnix + chance.integer({ min: 0, max: 86400 });
|
|
645
|
-
if (trialStart + timeShift < endUnix) {
|
|
646
|
-
subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart + timeShift, user, firstPaidPlan.name, 'trial'));
|
|
647
|
-
currentUnix = trialStart + (firstPaidPlan.trialDays * 86400);
|
|
648
|
-
|
|
649
|
-
// Trial to paid conversion
|
|
650
|
-
if (chance.bool({ likelihood: lc.trialToPayRate * personaUpgradeMod * 100 })) {
|
|
651
|
-
if (currentUnix + timeShift < endUnix) {
|
|
652
|
-
subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix + timeShift, user, firstPaidPlan.name, 'active'));
|
|
653
|
-
currentPlan = firstPaidPlan;
|
|
654
|
-
currentStatus = 'active';
|
|
655
|
-
}
|
|
656
|
-
} else {
|
|
657
|
-
currentStatus = 'expired_trial';
|
|
658
|
-
return subEvents; // didn't convert, no more sub events
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
// Monthly lifecycle loop
|
|
664
|
-
while (currentUnix + timeShift < endUnix) {
|
|
665
|
-
currentUnix += monthSeconds + chance.integer({ min: -86400, max: 86400 });
|
|
666
|
-
if (currentUnix + timeShift >= endUnix) break;
|
|
667
|
-
if (currentStatus === 'cancelled') {
|
|
668
|
-
// Win-back check
|
|
669
|
-
if (chance.bool({ likelihood: lc.winBackRate * 100 })) {
|
|
670
|
-
currentUnix += lc.winBackDelay * 86400;
|
|
671
|
-
if (currentUnix + timeShift >= endUnix) break;
|
|
672
|
-
subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix + timeShift, user, currentPlan.name, 'active'));
|
|
673
|
-
currentStatus = 'active';
|
|
674
|
-
}
|
|
675
|
-
break;
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
// Payment failure
|
|
679
|
-
if (currentPlan.price > 0 && chance.bool({ likelihood: lc.paymentFailureRate * 100 })) {
|
|
680
|
-
subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix + timeShift, user, currentPlan.name, 'payment_issue'));
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
// Churn
|
|
684
|
-
if (chance.bool({ likelihood: lc.churnRate * personaChurnMod * 100 })) {
|
|
685
|
-
subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix + timeShift, user, currentPlan.name, 'cancelled'));
|
|
686
|
-
currentStatus = 'cancelled';
|
|
687
|
-
continue;
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
// Upgrade
|
|
691
|
-
const currentPlanIndex = plans.indexOf(currentPlan);
|
|
692
|
-
if (currentPlanIndex < plans.length - 1 && chance.bool({ likelihood: lc.upgradeRate * personaUpgradeMod * 100 })) {
|
|
693
|
-
const newPlan = plans[currentPlanIndex + 1];
|
|
694
|
-
subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
|
|
695
|
-
currentPlan = newPlan;
|
|
696
|
-
continue;
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
// Downgrade
|
|
700
|
-
if (currentPlanIndex > 0 && currentPlan.price > 0 && chance.bool({ likelihood: lc.downgradeRate * 100 })) {
|
|
701
|
-
const newPlan = plans[currentPlanIndex - 1];
|
|
702
|
-
subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
|
|
703
|
-
currentPlan = newPlan;
|
|
704
|
-
continue;
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
// Renewal
|
|
708
|
-
if (currentPlan.price > 0) {
|
|
709
|
-
subEvents.push(makeSubEvent(eventNames.renewed, currentUnix + timeShift, user, currentPlan.name, 'active'));
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
return subEvents;
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
function makeSubEvent(eventName, unixTime, user, planName, status, previousPlan) {
|
|
717
|
-
const ev = {
|
|
718
|
-
event: eventName,
|
|
719
|
-
time: dayjs.unix(Math.min(unixTime, dayjs().unix())).toISOString(),
|
|
720
|
-
user_id: user.distinct_id,
|
|
721
|
-
insert_id: u.quickHash(`${eventName}-${unixTime}-${user.distinct_id}`),
|
|
722
|
-
subscription_plan: planName,
|
|
723
|
-
_currentPlan: planName,
|
|
724
|
-
_subStatus: status
|
|
725
|
-
};
|
|
726
|
-
if (previousPlan) ev.previous_plan = previousPlan;
|
|
727
|
-
return ev;
|
|
728
|
-
}
|
|
729
|
-
|
|
730
618
|
/**
|
|
731
619
|
* Feature 4: Generate bot users with repetitive patterns
|
|
732
620
|
*/
|
|
@@ -742,14 +630,14 @@ async function generateBotUsers(context, dataQuality, storage) {
|
|
|
742
630
|
const botId = `bot_${chance.guid().slice(0, 8)}`;
|
|
743
631
|
const botEvents = [];
|
|
744
632
|
|
|
745
|
-
// Bots generate events at machine-like intervals
|
|
746
|
-
let currentTime = context.FIXED_BEGIN
|
|
747
|
-
const interval = Math.floor(((context.
|
|
633
|
+
// Bots generate events at machine-like intervals across the dataset window
|
|
634
|
+
let currentTime = context.FIXED_BEGIN;
|
|
635
|
+
const interval = Math.floor(((context.FIXED_NOW - currentTime) / botEventsPerUser));
|
|
748
636
|
|
|
749
637
|
for (let e = 0; e < botEventsPerUser; e++) {
|
|
750
638
|
const chosenEvent = botEventTypes[e % botEventTypes.length];
|
|
751
639
|
currentTime += interval + chance.integer({ min: 0, max: 10 });
|
|
752
|
-
if (currentTime > context.
|
|
640
|
+
if (currentTime > context.FIXED_NOW) break;
|
|
753
641
|
botEvents.push({
|
|
754
642
|
event: chosenEvent.event,
|
|
755
643
|
time: dayjs.unix(currentTime).toISOString(),
|
|
@@ -771,41 +659,3 @@ async function generateBotUsers(context, dataQuality, storage) {
|
|
|
771
659
|
await storage.eventData.hookPush(botEvents, { profile: { distinct_id: botId, is_bot: true } });
|
|
772
660
|
}
|
|
773
661
|
}
|
|
774
|
-
|
|
775
|
-
/**
|
|
776
|
-
* Feature 9: Generate anomaly burst and coordinated events
|
|
777
|
-
*/
|
|
778
|
-
async function generateAnomalyBursts(context, anomalies, storage) {
|
|
779
|
-
const chance = u.getChance();
|
|
780
|
-
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
781
|
-
|
|
782
|
-
for (const a of anomalies) {
|
|
783
|
-
if (a.type !== 'burst' && a.type !== 'coordinated') continue;
|
|
784
|
-
if (!a._startUnix || !a.count) continue;
|
|
785
|
-
|
|
786
|
-
const burstEvents = [];
|
|
787
|
-
const startUnix = a._startUnix + timeShift;
|
|
788
|
-
const endUnix = a._endUnix + timeShift;
|
|
789
|
-
const windowSeconds = endUnix - startUnix;
|
|
790
|
-
|
|
791
|
-
for (let i = 0; i < a.count; i++) {
|
|
792
|
-
const eventTime = startUnix + chance.integer({ min: 0, max: Math.max(1, windowSeconds) });
|
|
793
|
-
if (eventTime > context.MAX_TIME) continue;
|
|
794
|
-
const userId = a.type === 'coordinated'
|
|
795
|
-
? `anomaly_${chance.guid().slice(0, 8)}`
|
|
796
|
-
: `burst_${chance.integer({ min: 1, max: 100 })}`;
|
|
797
|
-
|
|
798
|
-
const ev = {
|
|
799
|
-
event: a.event,
|
|
800
|
-
time: dayjs.unix(eventTime).toISOString(),
|
|
801
|
-
user_id: userId,
|
|
802
|
-
insert_id: u.quickHash(`${a.event}-${eventTime}-${userId}-${i}`),
|
|
803
|
-
};
|
|
804
|
-
if (a.tag) ev._anomaly = a.tag;
|
|
805
|
-
if (a.properties) Object.assign(ev, a.properties);
|
|
806
|
-
burstEvents.push(ev);
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
await storage.eventData.hookPush(burstEvents, {});
|
|
810
|
-
}
|
|
811
|
-
}
|