@ak--47/dungeon-master 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/dungeons/technical/ad-spend-schema.json +128 -0
- package/dungeons/technical/ad-spend.js +116 -0
- package/dungeons/technical/anonymous-users-schema.json +91 -0
- package/dungeons/technical/anonymous-users.js +102 -0
- package/dungeons/technical/array-of-object-lookup-schema.json +191 -0
- package/dungeons/{array-of-object-lookup.js → technical/array-of-object-lookup.js} +11 -4
- package/dungeons/technical/experiments-schema.json +197 -0
- package/dungeons/technical/experiments.js +168 -0
- package/dungeons/technical/foobar-schema.json +362 -0
- package/dungeons/{foobar.js → technical/foobar.js} +2 -2
- package/dungeons/technical/group-analytics-schema.json +149 -0
- package/dungeons/technical/group-analytics.js +151 -0
- package/dungeons/technical/mirror-strategies-schema.json +84 -0
- package/dungeons/technical/mirror-strategies.js +126 -0
- package/dungeons/technical/nested-objects-schema.json +145 -0
- package/dungeons/technical/nested-objects.js +207 -0
- package/dungeons/technical/retention-cadence-schema.json +37 -0
- package/dungeons/{retention-cadence.js → technical/retention-cadence.js} +1 -1
- package/dungeons/technical/sanity-schema.json +185 -0
- package/dungeons/{sanity.js → technical/sanity.js} +3 -2
- package/dungeons/technical/scale-test-schema.json +70 -0
- package/dungeons/technical/scale-test.js +90 -0
- package/dungeons/technical/scd-schema.json +270 -0
- package/dungeons/{scd.js → technical/scd.js} +19 -10
- package/dungeons/technical/simple-schema.json +278 -0
- package/dungeons/{simple.js → technical/simple.js} +6 -6
- package/dungeons/technical/simplest-schema.json +635 -0
- package/dungeons/{simplest.js → technical/simplest.js} +6 -6
- package/dungeons/technical/text-generation-schema.json +1059 -0
- package/dungeons/{text-generation.js → technical/text-generation.js} +23 -9
- package/dungeons/vertical/community-schema.json +557 -0
- package/dungeons/vertical/community.js +788 -0
- package/dungeons/vertical/devtools-schema.json +580 -0
- package/dungeons/vertical/devtools.js +837 -0
- package/dungeons/vertical/ecommerce-schema.json +586 -0
- package/dungeons/{ecommerce.js → vertical/ecommerce.js} +34 -3
- package/dungeons/vertical/education-schema.json +697 -0
- package/dungeons/{education.js → vertical/education.js} +54 -36
- package/dungeons/vertical/fintech-schema.json +563 -0
- package/dungeons/{fintech.js → vertical/fintech.js} +91 -53
- package/dungeons/vertical/fitness-schema.json +508 -0
- package/dungeons/vertical/fitness.js +802 -0
- package/dungeons/vertical/food-delivery-schema.json +678 -0
- package/dungeons/{food.js → vertical/food-delivery.js} +51 -23
- package/dungeons/vertical/gaming-schema.json +394 -0
- package/dungeons/{gaming.js → vertical/gaming.js} +9 -7
- package/dungeons/vertical/healthcare-schema.json +527 -0
- package/dungeons/vertical/healthcare.js +769 -0
- package/dungeons/vertical/insurance-application-schema.json +466 -0
- package/dungeons/{insurance-application.js → vertical/insurance-application.js} +33 -29
- package/dungeons/vertical/logistics-schema.json +552 -0
- package/dungeons/vertical/logistics.js +793 -0
- package/dungeons/vertical/marketplace-schema.json +511 -0
- package/dungeons/vertical/marketplace.js +844 -0
- package/dungeons/vertical/media-schema.json +787 -0
- package/dungeons/{media.js → vertical/media.js} +94 -16
- package/dungeons/vertical/rpg-schema.json +654 -0
- package/dungeons/{rpg.js → vertical/rpg.js} +78 -57
- package/dungeons/vertical/sass-schema.json +653 -0
- package/dungeons/{sass.js → vertical/sass.js} +49 -23
- package/dungeons/vertical/social-schema.json +552 -0
- package/dungeons/{social.js → vertical/social.js} +60 -14
- package/dungeons/vertical/travel-schema.json +558 -0
- package/dungeons/vertical/travel.js +736 -0
- package/index.js +19 -15
- package/lib/core/config-validator.js +286 -1
- package/lib/core/context.js +7 -5
- package/lib/generators/adspend.js +3 -3
- package/lib/generators/events.js +141 -14
- package/lib/generators/funnels.js +67 -7
- package/lib/generators/text.js +2 -2
- package/lib/orchestrators/user-loop.js +441 -22
- package/lib/templates/soup-presets.js +2 -2
- package/lib/utils/utils.js +4 -5
- package/package.json +4 -3
- package/scripts/dungeon-to-json.mjs +163 -0
- package/scripts/extract-dungeon-schema.mjs +297 -0
- package/scripts/json-to-dungeon.mjs +200 -0
- package/scripts/run-dungeon.mjs +76 -0
- package/scripts/run-many.mjs +118 -0
- package/scripts/verify-runner.mjs +42 -0
- package/types.d.ts +313 -1
- package/dungeons/array-of-object-lookup-schema.json +0 -327
- package/dungeons/ecommerce-schema.json +0 -462
- package/dungeons/education-schema.json +0 -2409
- package/dungeons/fintech-schema.json +0 -14034
- package/dungeons/foobar-schema.json +0 -403
- package/dungeons/food-delivery-schema.json +0 -192
- package/dungeons/food-delivery.js +0 -602
- package/dungeons/food-schema.json +0 -1152
- package/dungeons/gaming-schema.json +0 -1270
- package/dungeons/insurance-application-schema.json +0 -204
- package/dungeons/media-schema.json +0 -906
- package/dungeons/retention-cadence-schema.json +0 -78
- package/dungeons/rpg-schema.json +0 -4526
- package/dungeons/sanity-schema.json +0 -255
- package/dungeons/sass-schema.json +0 -1291
- package/dungeons/scd-schema.json +0 -919
- package/dungeons/simple-schema.json +0 -608
- package/dungeons/simplest-schema.json +0 -1418
- package/dungeons/social-schema.json +0 -1118
- package/dungeons/text-generation-schema.json +0 -3096
|
@@ -18,9 +18,10 @@ import { dataLogger as logger } from "../utils/logger.js";
|
|
|
18
18
|
* @param {number} firstEventTime - Unix timestamp for first event
|
|
19
19
|
* @param {Object} profile - User profile object
|
|
20
20
|
* @param {Object} scd - Slowly changing dimensions object
|
|
21
|
+
* @param {Object} [persona] - User's assigned persona
|
|
21
22
|
* @returns {Promise<[Array, boolean]>} Tuple of [events, didConvert]
|
|
22
23
|
*/
|
|
23
|
-
export async function makeFunnel(context, funnel, user, firstEventTime, profile = {}, scd = {}) {
|
|
24
|
+
export async function makeFunnel(context, funnel, user, firstEventTime, profile = {}, scd = {}, persona = null, featureCtx = {}) {
|
|
24
25
|
if (!funnel) throw new Error("no funnel");
|
|
25
26
|
if (!user) throw new Error("no user");
|
|
26
27
|
|
|
@@ -99,22 +100,65 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
99
100
|
const funnelPossibleEvents = buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, expName, expVariant);
|
|
100
101
|
|
|
101
102
|
// Handle repeat logic and conversion rate adjustment
|
|
102
|
-
|
|
103
|
+
let { processedEvents, adjustedConversionRate } = processEventRepeats(
|
|
103
104
|
funnelPossibleEvents,
|
|
104
105
|
requireRepeats,
|
|
105
106
|
conversionRate,
|
|
106
107
|
chance
|
|
107
108
|
);
|
|
108
109
|
|
|
110
|
+
// Apply persona conversion modifier (before hook, so hook can override via funnel-pre)
|
|
111
|
+
if (persona && persona.conversionModifier) {
|
|
112
|
+
adjustedConversionRate = Math.min(100, Math.max(0, Math.round(adjustedConversionRate * persona.conversionModifier)));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Apply world event conversion modifier if active at firstEventTime
|
|
116
|
+
const resolvedWorldEvents = /** @type {import('../../types').ResolvedWorldEvent[]} */ (config.worldEvents);
|
|
117
|
+
if (resolvedWorldEvents && firstEventTime) {
|
|
118
|
+
for (const we of resolvedWorldEvents) {
|
|
119
|
+
if (firstEventTime >= we.startUnix && firstEventTime < we.endUnix && we.conversionModifier !== 1.0) {
|
|
120
|
+
const affects = we.affectsEvents;
|
|
121
|
+
if (affects === "*" || (Array.isArray(affects) && sequence.some(s => affects.includes(s)))) {
|
|
122
|
+
adjustedConversionRate = Math.min(100, Math.max(0, Math.round(adjustedConversionRate * we.conversionModifier)));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Apply feature conversion lifts
|
|
129
|
+
const resolvedFeatures = config.features;
|
|
130
|
+
if (resolvedFeatures && firstEventTime) {
|
|
131
|
+
const daysSinceBegin = (firstEventTime - context.FIXED_BEGIN) / 86400;
|
|
132
|
+
for (const feat of resolvedFeatures) {
|
|
133
|
+
if (feat.conversionLift && daysSinceBegin >= feat.launchDay) {
|
|
134
|
+
const daysSinceLaunch = daysSinceBegin - feat.launchDay;
|
|
135
|
+
const { k, midpoint } = feat._resolvedCurve || { k: 0.08, midpoint: 30 };
|
|
136
|
+
const adoptionProb = 1 / (1 + Math.exp(-k * (daysSinceLaunch - midpoint)));
|
|
137
|
+
if (chance.bool({ likelihood: Math.min(100, adoptionProb * 100) })) {
|
|
138
|
+
adjustedConversionRate = Math.min(100, Math.max(0, Math.round(adjustedConversionRate * feat.conversionLift)));
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
109
144
|
// Determine if user converts and how many steps they'll take
|
|
145
|
+
// When experiment mode is active, $experiment_started is prepended to sequence
|
|
146
|
+
// but should not count as a funnel step for conversion purposes
|
|
147
|
+
const conversionStepCount = expName ? sequence.length - 1 : sequence.length;
|
|
110
148
|
const { doesUserConvert, numStepsUserWillTake } = determineConversion(
|
|
111
149
|
adjustedConversionRate,
|
|
112
|
-
|
|
150
|
+
conversionStepCount,
|
|
113
151
|
chance
|
|
114
152
|
);
|
|
115
153
|
|
|
116
154
|
// Get steps user will actually take
|
|
117
|
-
|
|
155
|
+
let funnelStepsUserWillTake;
|
|
156
|
+
if (expName) {
|
|
157
|
+
// $experiment_started always fires; conversion only applies to actual funnel steps
|
|
158
|
+
funnelStepsUserWillTake = [processedEvents[0], ...processedEvents.slice(1, 1 + numStepsUserWillTake)];
|
|
159
|
+
} else {
|
|
160
|
+
funnelStepsUserWillTake = processedEvents.slice(0, numStepsUserWillTake);
|
|
161
|
+
}
|
|
118
162
|
|
|
119
163
|
// Apply ordering strategy
|
|
120
164
|
const funnelActualOrder = applyOrderingStrategy(
|
|
@@ -137,6 +181,18 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
137
181
|
funnelEventsWithTiming.push(sessionStartEvent);
|
|
138
182
|
}
|
|
139
183
|
|
|
184
|
+
// Build complete feature context: merge passed-in featureCtx with config fallbacks
|
|
185
|
+
const funnelFeatureCtx = {
|
|
186
|
+
persona: featureCtx.persona || persona || null,
|
|
187
|
+
userCampaign: featureCtx.userCampaign || null,
|
|
188
|
+
userLocation: featureCtx.userLocation || null,
|
|
189
|
+
worldEventsTimeline: featureCtx.worldEventsTimeline || context.config.worldEvents || null,
|
|
190
|
+
resolvedFeatures: featureCtx.resolvedFeatures || context.config.features || null,
|
|
191
|
+
resolvedAnomalies: featureCtx.resolvedAnomalies || context.config.anomalies || null,
|
|
192
|
+
dataQuality: featureCtx.dataQuality || context.config.dataQuality || null,
|
|
193
|
+
geo: featureCtx.geo || context.config.geo || null,
|
|
194
|
+
};
|
|
195
|
+
|
|
140
196
|
// Generate actual events with timing
|
|
141
197
|
const finalEvents = await generateFunnelEvents(
|
|
142
198
|
context,
|
|
@@ -145,7 +201,8 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
145
201
|
firstEventTime || dayjs(created).unix(),
|
|
146
202
|
anonymousIds,
|
|
147
203
|
sessionIds,
|
|
148
|
-
groupKeys
|
|
204
|
+
groupKeys,
|
|
205
|
+
funnelFeatureCtx
|
|
149
206
|
);
|
|
150
207
|
|
|
151
208
|
// Call post-funnel hook
|
|
@@ -353,7 +410,8 @@ async function generateFunnelEvents(
|
|
|
353
410
|
earliestTime,
|
|
354
411
|
anonymousIds,
|
|
355
412
|
sessionIds,
|
|
356
|
-
groupKeys
|
|
413
|
+
groupKeys,
|
|
414
|
+
featureCtx = {}
|
|
357
415
|
) {
|
|
358
416
|
let funnelStartTime;
|
|
359
417
|
|
|
@@ -367,7 +425,9 @@ async function generateFunnelEvents(
|
|
|
367
425
|
sessionIds,
|
|
368
426
|
{},
|
|
369
427
|
groupKeys,
|
|
370
|
-
false // Let all funnel events use TimeSoup for proper time distribution
|
|
428
|
+
false, // Let all funnel events use TimeSoup for proper time distribution
|
|
429
|
+
false,
|
|
430
|
+
featureCtx
|
|
371
431
|
);
|
|
372
432
|
|
|
373
433
|
if (index === 0) {
|
package/lib/generators/text.js
CHANGED
|
@@ -542,9 +542,9 @@ class OrganicTextGenerator {
|
|
|
542
542
|
...config
|
|
543
543
|
};
|
|
544
544
|
|
|
545
|
-
// Initialize seed if provided
|
|
545
|
+
// Initialize seed if provided (use local RNG, not global, to avoid poisoning Math.random)
|
|
546
546
|
if (this.config.seed) {
|
|
547
|
-
seedrandom(this.config.seed
|
|
547
|
+
this._rng = seedrandom(this.config.seed);
|
|
548
548
|
}
|
|
549
549
|
|
|
550
550
|
// Initialize subsystems
|
|
@@ -42,6 +42,15 @@ export async function userLoop(context) {
|
|
|
42
42
|
percentUsersBornInDataset = 15,
|
|
43
43
|
strictEventCount = false,
|
|
44
44
|
bornRecentBias = 0.3, // 0 = uniform distribution, 1 = heavily biased toward recent births
|
|
45
|
+
personas,
|
|
46
|
+
worldEvents,
|
|
47
|
+
engagementDecay: globalEngagementDecay,
|
|
48
|
+
dataQuality,
|
|
49
|
+
subscription,
|
|
50
|
+
attribution,
|
|
51
|
+
geo,
|
|
52
|
+
features,
|
|
53
|
+
anomalies
|
|
45
54
|
} = config;
|
|
46
55
|
|
|
47
56
|
const { eventData, userProfilesData, scdTableData } = storage;
|
|
@@ -97,6 +106,19 @@ export async function userLoop(context) {
|
|
|
97
106
|
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
|
|
98
107
|
const { distinct_id, created } = user;
|
|
99
108
|
const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
|
|
109
|
+
|
|
110
|
+
// Feature 1: Assign persona
|
|
111
|
+
let persona = null;
|
|
112
|
+
if (personas && personas.length > 0) {
|
|
113
|
+
const weights = personas.map(p => p.weight);
|
|
114
|
+
const totalWeight = weights.reduce((a, b) => a + b, 0);
|
|
115
|
+
let roll = chance.floating({ min: 0, max: totalWeight });
|
|
116
|
+
for (const p of personas) {
|
|
117
|
+
roll -= p.weight;
|
|
118
|
+
if (roll <= 0) { persona = p; break; }
|
|
119
|
+
}
|
|
120
|
+
if (!persona) persona = personas[personas.length - 1];
|
|
121
|
+
}
|
|
100
122
|
let numEventsPreformed = 0;
|
|
101
123
|
|
|
102
124
|
if (!userIsBornInDataset) delete user.created;
|
|
@@ -112,7 +134,7 @@ export async function userLoop(context) {
|
|
|
112
134
|
|
|
113
135
|
if (bornRecentBias !== 0) {
|
|
114
136
|
// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
|
|
115
|
-
const datasetStart = dayjs.unix(
|
|
137
|
+
const datasetStart = dayjs.unix(context.FIXED_BEGIN);
|
|
116
138
|
const datasetEnd = dayjs.unix(context.FIXED_NOW);
|
|
117
139
|
const totalDuration = datasetEnd.diff(datasetStart);
|
|
118
140
|
// Clamp userPosition to [0, 1] to handle edge cases from rounding in time calculations
|
|
@@ -138,25 +160,113 @@ export async function userLoop(context) {
|
|
|
138
160
|
// Update user.created to match biased timestamp for profile consistency
|
|
139
161
|
user.created = adjustedCreated.toISOString();
|
|
140
162
|
} else {
|
|
141
|
-
adjustedCreated = dayjs.unix(
|
|
163
|
+
adjustedCreated = dayjs.unix(context.FIXED_BEGIN);
|
|
142
164
|
}
|
|
143
165
|
|
|
144
|
-
|
|
166
|
+
// Feature 7: Geographic intelligence — assign sticky location
|
|
167
|
+
let userLocation = null;
|
|
168
|
+
let userRegion = null;
|
|
169
|
+
let userTimezoneOffset = 0;
|
|
170
|
+
if (geo && geo.sticky && geo.regions && geo.regions.length > 0) {
|
|
171
|
+
// Assign region using weighted selection
|
|
172
|
+
const regionWeights = geo.regions.map(r => r.weight);
|
|
173
|
+
const totalRegionWeight = regionWeights.reduce((a, b) => a + b, 0);
|
|
174
|
+
let regionRoll = chance.floating({ min: 0, max: totalRegionWeight });
|
|
175
|
+
for (const r of geo.regions) {
|
|
176
|
+
regionRoll -= r.weight;
|
|
177
|
+
if (regionRoll <= 0) { userRegion = r; break; }
|
|
178
|
+
}
|
|
179
|
+
if (!userRegion) userRegion = geo.regions[geo.regions.length - 1];
|
|
180
|
+
userTimezoneOffset = userRegion.timezoneOffset || 0;
|
|
181
|
+
|
|
182
|
+
// Pick a location matching one of this region's countries
|
|
183
|
+
const regionLocations = u.choose(defaults.locationsUsers).filter(
|
|
184
|
+
loc => userRegion.countries.includes(loc.country_code || loc.country)
|
|
185
|
+
);
|
|
186
|
+
if (regionLocations.length > 0) {
|
|
187
|
+
userLocation = u.pickRandom(regionLocations);
|
|
188
|
+
} else {
|
|
189
|
+
userLocation = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
190
|
+
}
|
|
191
|
+
for (const key in userLocation) {
|
|
192
|
+
user[key] = userLocation[key];
|
|
193
|
+
}
|
|
194
|
+
// Inject region properties
|
|
195
|
+
if (userRegion.properties) {
|
|
196
|
+
for (const [k, v] of Object.entries(userRegion.properties)) {
|
|
197
|
+
user[k] = v;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
} else if (hasLocation) {
|
|
145
201
|
const location = u.pickRandom(u.choose(defaults.locationsUsers));
|
|
146
202
|
for (const key in location) {
|
|
147
203
|
user[key] = location[key];
|
|
148
204
|
}
|
|
205
|
+
userLocation = location;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Feature 6: Attribution — assign campaign to users born in dataset
|
|
209
|
+
let userCampaign = null;
|
|
210
|
+
if (attribution && userIsBornInDataset) {
|
|
211
|
+
// adjustedCreated is in the internal FIXED time range
|
|
212
|
+
const birthUnix = adjustedCreated.unix();
|
|
213
|
+
const birthDay = Math.max(0, (birthUnix - context.FIXED_BEGIN) / 86400);
|
|
214
|
+
const isOrganic = chance.bool({ likelihood: (attribution.organicRate || 0.4) * 100 });
|
|
215
|
+
if (!isOrganic) {
|
|
216
|
+
// Find active campaigns at birth day
|
|
217
|
+
const activeCampaigns = attribution.campaigns.filter(c =>
|
|
218
|
+
birthDay >= c.activeDays[0] && birthDay <= c.activeDays[1]
|
|
219
|
+
);
|
|
220
|
+
if (activeCampaigns.length > 0) {
|
|
221
|
+
userCampaign = chance.pickone(activeCampaigns);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
149
224
|
}
|
|
150
225
|
|
|
151
226
|
// Profile creation
|
|
152
227
|
const profile = await makeUserProfile(context, userProps, user);
|
|
153
228
|
|
|
154
|
-
//
|
|
229
|
+
// Feature 1: Merge persona properties into profile (before hook, so hook can override)
|
|
230
|
+
if (persona && persona.properties) {
|
|
231
|
+
for (const [key, value] of Object.entries(persona.properties)) {
|
|
232
|
+
profile[key] = u.choose(value);
|
|
233
|
+
}
|
|
234
|
+
profile._persona = persona.name;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Feature 6: Add attribution to profile
|
|
238
|
+
if (userCampaign) {
|
|
239
|
+
profile.utm_source = userCampaign.source;
|
|
240
|
+
profile.utm_campaign = userCampaign.name;
|
|
241
|
+
if (userCampaign.medium) profile.utm_medium = userCampaign.medium;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Feature 7: Add region to profile
|
|
245
|
+
if (userRegion) {
|
|
246
|
+
profile._region = userRegion.name;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Build feature context for event generation
|
|
250
|
+
const featureCtx = {
|
|
251
|
+
persona,
|
|
252
|
+
userLocation,
|
|
253
|
+
worldEventsTimeline: worldEvents,
|
|
254
|
+
resolvedFeatures: features,
|
|
255
|
+
resolvedAnomalies: anomalies,
|
|
256
|
+
dataQuality,
|
|
257
|
+
geo,
|
|
258
|
+
userCampaign,
|
|
259
|
+
userRegion,
|
|
260
|
+
userTimezoneOffset
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
// Call user hook after profile creation (hooks override persona properties)
|
|
155
264
|
if (config.hook) {
|
|
156
265
|
await config.hook(profile, "user", {
|
|
157
266
|
user,
|
|
158
267
|
config,
|
|
159
|
-
userIsBornInDataset
|
|
268
|
+
userIsBornInDataset,
|
|
269
|
+
persona
|
|
160
270
|
});
|
|
161
271
|
}
|
|
162
272
|
|
|
@@ -172,16 +282,18 @@ export async function userLoop(context) {
|
|
|
172
282
|
let changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
|
|
173
283
|
userSCD[key] = changes;
|
|
174
284
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
285
|
+
if (config.hook) {
|
|
286
|
+
const hookResult = await config.hook(changes, "scd-pre", {
|
|
287
|
+
profile,
|
|
288
|
+
type: 'user',
|
|
289
|
+
scd: { [key]: scdProps[key] },
|
|
290
|
+
config,
|
|
291
|
+
allSCDs: userSCD
|
|
292
|
+
});
|
|
293
|
+
if (Array.isArray(hookResult)) {
|
|
294
|
+
changes = hookResult;
|
|
295
|
+
userSCD[key] = changes;
|
|
296
|
+
}
|
|
185
297
|
}
|
|
186
298
|
}
|
|
187
299
|
|
|
@@ -191,8 +303,14 @@ export async function userLoop(context) {
|
|
|
191
303
|
}) * 0.714159265359);
|
|
192
304
|
|
|
193
305
|
// Power users and low-activity users logic
|
|
194
|
-
|
|
195
|
-
|
|
306
|
+
if (persona) {
|
|
307
|
+
// Persona-driven event multiplier replaces the old dice rolls
|
|
308
|
+
numEventsThisUserWillPreform *= persona.eventMultiplier;
|
|
309
|
+
} else {
|
|
310
|
+
// Legacy behavior when no personas configured
|
|
311
|
+
chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
|
|
312
|
+
chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
|
|
313
|
+
}
|
|
196
314
|
numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
|
|
197
315
|
|
|
198
316
|
let userFirstEventTime;
|
|
@@ -228,7 +346,7 @@ export async function userLoop(context) {
|
|
|
228
346
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
229
347
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
230
348
|
const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
231
|
-
const [data, converted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
|
|
349
|
+
const [data, converted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD, persona, featureCtx);
|
|
232
350
|
userConverted = converted;
|
|
233
351
|
|
|
234
352
|
const timeShift = context.getTimeShift();
|
|
@@ -245,11 +363,11 @@ export async function userLoop(context) {
|
|
|
245
363
|
let newEvents;
|
|
246
364
|
if (usageFunnels.length && userConverted) {
|
|
247
365
|
const currentFunnel = chance.pickone(usageFunnels);
|
|
248
|
-
const [data, converted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
|
|
366
|
+
const [data, converted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD, persona, featureCtx);
|
|
249
367
|
numEventsPreformed += data.length;
|
|
250
368
|
newEvents = data;
|
|
251
369
|
} else {
|
|
252
|
-
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
|
|
370
|
+
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true, false, featureCtx);
|
|
253
371
|
numEventsPreformed++;
|
|
254
372
|
newEvents = [data];
|
|
255
373
|
}
|
|
@@ -275,13 +393,66 @@ export async function userLoop(context) {
|
|
|
275
393
|
// Remove events flagged as future timestamps (before dungeon hooks see them)
|
|
276
394
|
usersEvents = usersEvents.filter(e => !e._drop);
|
|
277
395
|
|
|
278
|
-
//
|
|
396
|
+
// Feature 3: Engagement decay — filter behavioral events BEFORE subscription injection
|
|
397
|
+
// Subscription events (trial, upgrade, cancel) must not be randomly dropped by decay
|
|
398
|
+
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
399
|
+
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
400
|
+
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Feature 5: Subscription lifecycle — inject after decay (exempt from decay filtering)
|
|
404
|
+
if (subscription && userIsBornInDataset) {
|
|
405
|
+
const subEvents = generateSubscriptionEvents(
|
|
406
|
+
subscription, user, persona, adjustedCreated, context, chance
|
|
407
|
+
);
|
|
408
|
+
if (subEvents.length > 0) {
|
|
409
|
+
usersEvents = usersEvents.concat(subEvents);
|
|
410
|
+
// Perf 3: ISO strings sort lexicographically — avoid Date() allocation
|
|
411
|
+
usersEvents.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
412
|
+
}
|
|
413
|
+
// Set current plan on profile
|
|
414
|
+
if (subEvents.length > 0) {
|
|
415
|
+
const lastSubEvent = subEvents[subEvents.length - 1];
|
|
416
|
+
if (lastSubEvent._currentPlan) {
|
|
417
|
+
profile.subscription_plan = lastSubEvent._currentPlan;
|
|
418
|
+
profile.subscription_status = lastSubEvent._subStatus || 'active';
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// Feature 4: Data quality — duplicates and late-arriving
|
|
424
|
+
if (dataQuality) {
|
|
425
|
+
if (dataQuality.duplicateRate > 0) {
|
|
426
|
+
const dupes = [];
|
|
427
|
+
for (const ev of usersEvents) {
|
|
428
|
+
if (chance.bool({ likelihood: dataQuality.duplicateRate * 100 })) {
|
|
429
|
+
const dupe = { ...ev };
|
|
430
|
+
dupe.time = dayjs(ev.time).add(chance.integer({ min: 1, max: 60 }), 'seconds').toISOString();
|
|
431
|
+
// Fix 2: Regenerate insert_id so Mixpanel doesn't silently deduplicate
|
|
432
|
+
const dupeId = dupe.user_id || dupe.device_id || '';
|
|
433
|
+
dupe.insert_id = u.quickHash(`${dupe.event}-${dupe.time}-${dupeId}-dupe`);
|
|
434
|
+
dupes.push(dupe);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
usersEvents = usersEvents.concat(dupes);
|
|
438
|
+
}
|
|
439
|
+
if (dataQuality.lateArrivingRate > 0) {
|
|
440
|
+
for (const ev of usersEvents) {
|
|
441
|
+
if (chance.bool({ likelihood: dataQuality.lateArrivingRate * 100 })) {
|
|
442
|
+
ev.time = dayjs(ev.time).subtract(chance.integer({ min: 1, max: 7 }), 'days').toISOString();
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Hook for processing all user events (hooks override everything)
|
|
279
449
|
if (config.hook) {
|
|
280
450
|
const newEvents = await config.hook(usersEvents, "everything", {
|
|
281
451
|
profile,
|
|
282
452
|
scd: userSCD,
|
|
283
453
|
config,
|
|
284
|
-
userIsBornInDataset
|
|
454
|
+
userIsBornInDataset,
|
|
455
|
+
persona
|
|
285
456
|
});
|
|
286
457
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
287
458
|
}
|
|
@@ -314,6 +485,16 @@ export async function userLoop(context) {
|
|
|
314
485
|
// Wait for all users to complete
|
|
315
486
|
await Promise.all(userPromises);
|
|
316
487
|
|
|
488
|
+
// Feature 4: Generate bot users (after regular users)
|
|
489
|
+
if (dataQuality && dataQuality.botUsers > 0) {
|
|
490
|
+
await generateBotUsers(context, dataQuality, storage);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// Feature 9: Anomaly burst/coordinated injection (after all users)
|
|
494
|
+
if (anomalies) {
|
|
495
|
+
await generateAnomalyBursts(context, anomalies, storage);
|
|
496
|
+
}
|
|
497
|
+
|
|
317
498
|
// Clean up SIGINT handler
|
|
318
499
|
process.removeListener('SIGINT', onSigint);
|
|
319
500
|
}
|
|
@@ -333,3 +514,241 @@ export function matchConditions(profile, conditions) {
|
|
|
333
514
|
}
|
|
334
515
|
return true;
|
|
335
516
|
}
|
|
517
|
+
|
|
518
|
+
// ── Advanced Feature Helper Functions ──
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Feature 3: Apply engagement decay to a user's events
|
|
522
|
+
*/
|
|
523
|
+
function applyEngagementDecay(events, decay, userCreated, context, chance) {
|
|
524
|
+
if (!events.length) return events;
|
|
525
|
+
// Perf 2: Use Date.parse instead of dayjs for hot loop
|
|
526
|
+
const userStartUnix = new Date(userCreated.toISOString ? userCreated.toISOString() : userCreated).getTime() / 1000;
|
|
527
|
+
const halfLifeDays = decay.halfLife || 45;
|
|
528
|
+
const floor = decay.floor ?? 0.1;
|
|
529
|
+
const reactivationChance = decay.reactivationChance || 0;
|
|
530
|
+
const reactivationMult = decay.reactivationMultiplier || 2.0;
|
|
531
|
+
|
|
532
|
+
return events.filter(ev => {
|
|
533
|
+
const evUnix = new Date(ev.time).getTime() / 1000;
|
|
534
|
+
const daysSinceBirth = Math.max(0, (evUnix - userStartUnix) / 86400);
|
|
535
|
+
|
|
536
|
+
let retention;
|
|
537
|
+
switch (decay.model) {
|
|
538
|
+
case 'exponential':
|
|
539
|
+
retention = Math.max(floor, Math.pow(0.5, daysSinceBirth / halfLifeDays));
|
|
540
|
+
break;
|
|
541
|
+
case 'linear':
|
|
542
|
+
retention = Math.max(floor, 1 - (daysSinceBirth / (halfLifeDays * 2)));
|
|
543
|
+
break;
|
|
544
|
+
case 'step':
|
|
545
|
+
retention = daysSinceBirth < halfLifeDays ? 1.0 : floor;
|
|
546
|
+
break;
|
|
547
|
+
default:
|
|
548
|
+
return true;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// Reactivation chance can spike retention back up
|
|
552
|
+
if (reactivationChance > 0 && retention < 0.5 && chance.bool({ likelihood: reactivationChance * 100 })) {
|
|
553
|
+
retention = Math.min(1.0, retention * reactivationMult);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
return chance.bool({ likelihood: retention * 100 });
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Feature 5: Generate subscription lifecycle events for a user
|
|
562
|
+
*/
|
|
563
|
+
function generateSubscriptionEvents(subscription, user, persona, userCreated, context, chance) {
|
|
564
|
+
const { plans, lifecycle, events: eventNames } = subscription;
|
|
565
|
+
const lc = lifecycle;
|
|
566
|
+
const subEvents = [];
|
|
567
|
+
|
|
568
|
+
const defaultPlan = plans.find(p => p.default) || plans[0];
|
|
569
|
+
let currentPlan = defaultPlan;
|
|
570
|
+
let currentStatus = 'active';
|
|
571
|
+
const paidPlans = plans.filter(p => p.price > 0);
|
|
572
|
+
|
|
573
|
+
const userStartUnix = dayjs(userCreated).unix();
|
|
574
|
+
const endUnix = context.MAX_TIME;
|
|
575
|
+
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
576
|
+
|
|
577
|
+
// Persona modifiers for subscription behavior
|
|
578
|
+
const personaChurnMod = persona?.churnRate ? (1 + persona.churnRate) : 1.0;
|
|
579
|
+
const personaUpgradeMod = persona?.conversionModifier || 1.0;
|
|
580
|
+
|
|
581
|
+
let currentUnix = userStartUnix;
|
|
582
|
+
const monthSeconds = 30 * 86400;
|
|
583
|
+
|
|
584
|
+
// If the default plan has a trial, start a trial
|
|
585
|
+
const firstPaidPlan = paidPlans[0];
|
|
586
|
+
if (firstPaidPlan && firstPaidPlan.trialDays) {
|
|
587
|
+
const trialStart = currentUnix + chance.integer({ min: 0, max: 86400 });
|
|
588
|
+
if (trialStart + timeShift < endUnix) {
|
|
589
|
+
subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart + timeShift, user, firstPaidPlan.name, 'trial'));
|
|
590
|
+
currentUnix = trialStart + (firstPaidPlan.trialDays * 86400);
|
|
591
|
+
|
|
592
|
+
// Trial to paid conversion
|
|
593
|
+
if (chance.bool({ likelihood: lc.trialToPayRate * personaUpgradeMod * 100 })) {
|
|
594
|
+
if (currentUnix + timeShift < endUnix) {
|
|
595
|
+
subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix + timeShift, user, firstPaidPlan.name, 'active'));
|
|
596
|
+
currentPlan = firstPaidPlan;
|
|
597
|
+
currentStatus = 'active';
|
|
598
|
+
}
|
|
599
|
+
} else {
|
|
600
|
+
currentStatus = 'expired_trial';
|
|
601
|
+
return subEvents; // didn't convert, no more sub events
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Monthly lifecycle loop
|
|
607
|
+
while (currentUnix + timeShift < endUnix) {
|
|
608
|
+
currentUnix += monthSeconds + chance.integer({ min: -86400, max: 86400 });
|
|
609
|
+
if (currentUnix + timeShift >= endUnix) break;
|
|
610
|
+
if (currentStatus === 'cancelled') {
|
|
611
|
+
// Win-back check
|
|
612
|
+
if (chance.bool({ likelihood: lc.winBackRate * 100 })) {
|
|
613
|
+
currentUnix += lc.winBackDelay * 86400;
|
|
614
|
+
if (currentUnix + timeShift >= endUnix) break;
|
|
615
|
+
subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix + timeShift, user, currentPlan.name, 'active'));
|
|
616
|
+
currentStatus = 'active';
|
|
617
|
+
}
|
|
618
|
+
break;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// Payment failure
|
|
622
|
+
if (currentPlan.price > 0 && chance.bool({ likelihood: lc.paymentFailureRate * 100 })) {
|
|
623
|
+
subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix + timeShift, user, currentPlan.name, 'payment_issue'));
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Churn
|
|
627
|
+
if (chance.bool({ likelihood: lc.churnRate * personaChurnMod * 100 })) {
|
|
628
|
+
subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix + timeShift, user, currentPlan.name, 'cancelled'));
|
|
629
|
+
currentStatus = 'cancelled';
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// Upgrade
|
|
634
|
+
const currentPlanIndex = plans.indexOf(currentPlan);
|
|
635
|
+
if (currentPlanIndex < plans.length - 1 && chance.bool({ likelihood: lc.upgradeRate * personaUpgradeMod * 100 })) {
|
|
636
|
+
const newPlan = plans[currentPlanIndex + 1];
|
|
637
|
+
subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
|
|
638
|
+
currentPlan = newPlan;
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// Downgrade
|
|
643
|
+
if (currentPlanIndex > 0 && currentPlan.price > 0 && chance.bool({ likelihood: lc.downgradeRate * 100 })) {
|
|
644
|
+
const newPlan = plans[currentPlanIndex - 1];
|
|
645
|
+
subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix + timeShift, user, newPlan.name, 'active', currentPlan.name));
|
|
646
|
+
currentPlan = newPlan;
|
|
647
|
+
continue;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Renewal
|
|
651
|
+
if (currentPlan.price > 0) {
|
|
652
|
+
subEvents.push(makeSubEvent(eventNames.renewed, currentUnix + timeShift, user, currentPlan.name, 'active'));
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
return subEvents;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
function makeSubEvent(eventName, unixTime, user, planName, status, previousPlan) {
|
|
660
|
+
const ev = {
|
|
661
|
+
event: eventName,
|
|
662
|
+
time: dayjs.unix(Math.min(unixTime, dayjs().unix())).toISOString(),
|
|
663
|
+
user_id: user.distinct_id,
|
|
664
|
+
insert_id: u.quickHash(`${eventName}-${unixTime}-${user.distinct_id}`),
|
|
665
|
+
subscription_plan: planName,
|
|
666
|
+
_currentPlan: planName,
|
|
667
|
+
_subStatus: status
|
|
668
|
+
};
|
|
669
|
+
if (previousPlan) ev.previous_plan = previousPlan;
|
|
670
|
+
return ev;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Feature 4: Generate bot users with repetitive patterns
|
|
675
|
+
*/
|
|
676
|
+
async function generateBotUsers(context, dataQuality, storage) {
|
|
677
|
+
const { botUsers, botEventsPerUser } = dataQuality;
|
|
678
|
+
const chance = u.getChance();
|
|
679
|
+
const { config } = context;
|
|
680
|
+
const events = config.events || [{ event: 'page_view' }];
|
|
681
|
+
// Bots use only 1-2 event types repetitively
|
|
682
|
+
const botEventTypes = events.slice(0, Math.min(2, events.length));
|
|
683
|
+
|
|
684
|
+
for (let b = 0; b < botUsers; b++) {
|
|
685
|
+
const botId = `bot_${chance.guid().slice(0, 8)}`;
|
|
686
|
+
const botEvents = [];
|
|
687
|
+
|
|
688
|
+
// Bots generate events at machine-like intervals
|
|
689
|
+
let currentTime = context.FIXED_BEGIN + context.TIME_SHIFT_SECONDS;
|
|
690
|
+
const interval = Math.floor(((context.MAX_TIME - currentTime) / botEventsPerUser));
|
|
691
|
+
|
|
692
|
+
for (let e = 0; e < botEventsPerUser; e++) {
|
|
693
|
+
const chosenEvent = botEventTypes[e % botEventTypes.length];
|
|
694
|
+
currentTime += interval + chance.integer({ min: 0, max: 10 });
|
|
695
|
+
if (currentTime > context.MAX_TIME) break;
|
|
696
|
+
botEvents.push({
|
|
697
|
+
event: chosenEvent.event,
|
|
698
|
+
time: dayjs.unix(currentTime).toISOString(),
|
|
699
|
+
user_id: botId,
|
|
700
|
+
device_id: botId,
|
|
701
|
+
insert_id: u.quickHash(`${chosenEvent.event}-${currentTime}-${botId}`),
|
|
702
|
+
is_bot: true
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// Store bot profile
|
|
707
|
+
await storage.userProfilesData.hookPush({
|
|
708
|
+
distinct_id: botId,
|
|
709
|
+
name: `Bot ${b + 1}`,
|
|
710
|
+
is_bot: true
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
// Store bot events
|
|
714
|
+
await storage.eventData.hookPush(botEvents, { profile: { distinct_id: botId, is_bot: true } });
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Feature 9: Generate anomaly burst and coordinated events
|
|
720
|
+
*/
|
|
721
|
+
async function generateAnomalyBursts(context, anomalies, storage) {
|
|
722
|
+
const chance = u.getChance();
|
|
723
|
+
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
724
|
+
|
|
725
|
+
for (const a of anomalies) {
|
|
726
|
+
if (a.type !== 'burst' && a.type !== 'coordinated') continue;
|
|
727
|
+
if (!a._startUnix || !a.count) continue;
|
|
728
|
+
|
|
729
|
+
const burstEvents = [];
|
|
730
|
+
const startUnix = a._startUnix + timeShift;
|
|
731
|
+
const endUnix = a._endUnix + timeShift;
|
|
732
|
+
const windowSeconds = endUnix - startUnix;
|
|
733
|
+
|
|
734
|
+
for (let i = 0; i < a.count; i++) {
|
|
735
|
+
const eventTime = startUnix + chance.integer({ min: 0, max: Math.max(1, windowSeconds) });
|
|
736
|
+
if (eventTime > context.MAX_TIME) continue;
|
|
737
|
+
const userId = a.type === 'coordinated'
|
|
738
|
+
? `anomaly_${chance.guid().slice(0, 8)}`
|
|
739
|
+
: `burst_${chance.integer({ min: 1, max: 100 })}`;
|
|
740
|
+
|
|
741
|
+
const ev = {
|
|
742
|
+
event: a.event,
|
|
743
|
+
time: dayjs.unix(eventTime).toISOString(),
|
|
744
|
+
user_id: userId,
|
|
745
|
+
insert_id: u.quickHash(`${a.event}-${eventTime}-${userId}-${i}`),
|
|
746
|
+
};
|
|
747
|
+
if (a.tag) ev._anomaly = a.tag;
|
|
748
|
+
if (a.properties) Object.assign(ev, a.properties);
|
|
749
|
+
burstEvents.push(ev);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
await storage.eventData.hookPush(burstEvents, {});
|
|
753
|
+
}
|
|
754
|
+
}
|