@ak--47/dungeon-master 1.3.1 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dungeons/technical/hook-helpers-verify.js +89 -0
- package/dungeons/technical/identity-model-verify.js +47 -0
- package/dungeons/technical/pattern-aggregate-by-bin.js +41 -0
- package/dungeons/technical/pattern-attributed-by-source.js +42 -0
- package/dungeons/technical/pattern-frequency-by-frequency.js +40 -0
- package/dungeons/technical/pattern-funnel-frequency.js +54 -0
- package/dungeons/technical/pattern-ttc-by-segment.js +45 -0
- package/dungeons/vertical/ai-platform.js +45 -52
- package/dungeons/vertical/community.js +11 -8
- package/dungeons/vertical/crypto.js +25 -24
- package/dungeons/vertical/dating.js +56 -48
- package/dungeons/vertical/devtools.js +25 -18
- package/dungeons/vertical/ecommerce.js +42 -38
- package/dungeons/vertical/education.js +24 -9
- package/dungeons/vertical/fintech.js +13 -8
- package/dungeons/vertical/fitness.js +73 -122
- package/dungeons/vertical/food-delivery.js +18 -19
- package/dungeons/vertical/gaming.js +19 -20
- package/dungeons/vertical/healthcare.js +11 -8
- package/dungeons/vertical/insurance-application.js +6 -3
- package/dungeons/vertical/logistics.js +15 -9
- package/dungeons/vertical/marketplace.js +36 -27
- package/dungeons/vertical/media.js +27 -25
- package/dungeons/vertical/real-estate.js +18 -7
- package/dungeons/vertical/sass.js +84 -68
- package/dungeons/vertical/social.js +46 -47
- package/dungeons/vertical/travel.js +8 -5
- package/index.js +17 -71
- package/lib/core/config-validator.js +143 -164
- package/lib/core/storage.js +5 -1
- package/lib/generators/events.js +49 -93
- package/lib/generators/funnels.js +202 -91
- package/lib/hook-helpers/_internal.js +23 -0
- package/lib/hook-helpers/cohort.js +124 -0
- package/lib/hook-helpers/identity.js +56 -0
- package/lib/hook-helpers/index.js +44 -0
- package/lib/hook-helpers/inject.js +99 -0
- package/lib/hook-helpers/mutate.js +151 -0
- package/lib/hook-helpers/timing.js +99 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +38 -0
- package/lib/hook-patterns/attributed-by-source.js +72 -0
- package/lib/hook-patterns/frequency-by-frequency.js +46 -0
- package/lib/hook-patterns/funnel-frequency-breakdown.js +73 -0
- package/lib/hook-patterns/index.js +14 -0
- package/lib/hook-patterns/time-to-convert-by-segment.js +41 -0
- package/lib/orchestrators/mixpanel-sender.js +46 -51
- package/lib/orchestrators/user-loop.js +119 -269
- package/lib/utils/utils.js +39 -16
- package/lib/verify/emulate-breakdown.js +281 -0
- package/lib/verify/index.js +12 -0
- package/lib/verify/verify-dungeon.js +61 -0
- package/package.json +6 -4
- package/types.d.ts +404 -212
|
@@ -30,7 +30,7 @@ import { resolveMacro } from "../templates/macro-presets.js";
|
|
|
30
30
|
* @param {number} [userNumDays]
|
|
31
31
|
* @returns {{ datasetStartUnix: number, datasetEndUnix: number, numDays: number }}
|
|
32
32
|
*/
|
|
33
|
-
function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays) {
|
|
33
|
+
function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays, verbose = false) {
|
|
34
34
|
const hasStart = datasetStart !== undefined && datasetStart !== null;
|
|
35
35
|
const hasEnd = datasetEnd !== undefined && datasetEnd !== null;
|
|
36
36
|
|
|
@@ -48,7 +48,7 @@ function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays) {
|
|
|
48
48
|
throw new Error(`datasetEnd (${datasetEnd}) must be after datasetStart (${datasetStart}).`);
|
|
49
49
|
}
|
|
50
50
|
const derivedNumDays = Math.max(1, Math.round((endUnix - startUnix) / 86400));
|
|
51
|
-
if (userNumDays !== undefined && userNumDays !== null && userNumDays !== derivedNumDays) {
|
|
51
|
+
if (verbose && userNumDays !== undefined && userNumDays !== null && userNumDays !== derivedNumDays) {
|
|
52
52
|
console.warn(
|
|
53
53
|
`⚠️ datasetStart/datasetEnd take precedence; user-supplied numDays=${userNumDays} ignored, derived numDays=${derivedNumDays}.`
|
|
54
54
|
);
|
|
@@ -60,7 +60,7 @@ function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays) {
|
|
|
60
60
|
const fallbackNumDays = (typeof userNumDays === 'number' && userNumDays > 0) ? userNumDays : 30;
|
|
61
61
|
const todayStart = dayjs().startOf('day').unix();
|
|
62
62
|
const fallbackStart = todayStart - fallbackNumDays * 86400;
|
|
63
|
-
console.warn(
|
|
63
|
+
if (verbose) console.warn(
|
|
64
64
|
`⚠️ No 'datasetStart'/'datasetEnd' set — dataset window anchored to today's date and will shift across runs. Pin both for full determinism.`
|
|
65
65
|
);
|
|
66
66
|
return { datasetStartUnix: fallbackStart, datasetEndUnix: todayStart, numDays: fallbackNumDays };
|
|
@@ -145,6 +145,106 @@ function inferFunnels(events) {
|
|
|
145
145
|
return createdFunnels;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
/**
|
|
149
|
+
* Config keys removed from the engine in 1.4. Silently stripped by
|
|
150
|
+
* `validateDungeonConfig` with a single warning per dungeon. To recreate any of these
|
|
151
|
+
* patterns, use hooks (see `lib/hook-patterns/*`).
|
|
152
|
+
*/
|
|
153
|
+
const KILLED_CONFIG_KEYS = ['subscription', 'attribution', 'geo', 'features', 'anomalies'];
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Strip killed config keys in place, log one deprecation warning per dungeon.
|
|
157
|
+
* @param {Partial<Dungeon>} config
|
|
158
|
+
*/
|
|
159
|
+
function stripKilledConfigKeys(config) {
|
|
160
|
+
const found = KILLED_CONFIG_KEYS.filter(k => config[k] !== undefined && config[k] !== null);
|
|
161
|
+
if (!found.length) return;
|
|
162
|
+
for (const k of found) delete config[k];
|
|
163
|
+
if (config.verbose) {
|
|
164
|
+
console.warn(
|
|
165
|
+
`⚠️ dungeon-master 1.4 removed engine support for: ${found.join(', ')}. ` +
|
|
166
|
+
`These config keys are silently ignored. Recreate via hooks (see lib/hook-patterns/* once Phase 4 lands).`
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Validate `Funnel.attempts` config in place. Coerces missing/invalid bounds so callers
|
|
173
|
+
* downstream don't have to re-defend. Throws on logically invalid configs (max < min).
|
|
174
|
+
* @param {import('../../types.js').Funnel[]} funnels
|
|
175
|
+
*/
|
|
176
|
+
function validateAttempts(funnels) {
|
|
177
|
+
for (const f of funnels) {
|
|
178
|
+
if (!f || !f.attempts) continue;
|
|
179
|
+
const a = f.attempts;
|
|
180
|
+
const min = Number.isFinite(a.min) ? Math.max(0, Math.floor(a.min)) : 0;
|
|
181
|
+
const max = Number.isFinite(a.max) ? Math.max(0, Math.floor(a.max)) : min;
|
|
182
|
+
if (max < min) {
|
|
183
|
+
throw new Error(`Funnel "${f.name || f.sequence?.join(' > ')}" attempts.max (${a.max}) must be >= attempts.min (${a.min})`);
|
|
184
|
+
}
|
|
185
|
+
a.min = min;
|
|
186
|
+
a.max = max;
|
|
187
|
+
if (a.conversionRate !== undefined) {
|
|
188
|
+
if (!Number.isFinite(a.conversionRate)) {
|
|
189
|
+
throw new Error(`Funnel "${f.name || f.sequence?.join(' > ')}" attempts.conversionRate must be a finite number 0-100`);
|
|
190
|
+
}
|
|
191
|
+
a.conversionRate = Math.max(0, Math.min(100, a.conversionRate));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Normalize `Funnel.experiment` in place. `true` → default 3-variant config.
|
|
198
|
+
* Object → validated and resolved. Stores `funnel._experiment` for downstream.
|
|
199
|
+
* @param {import('../../types.js').Funnel[]} funnels
|
|
200
|
+
* @param {number} datasetEndUnix
|
|
201
|
+
*/
|
|
202
|
+
function normalizeExperiments(funnels, datasetEndUnix) {
|
|
203
|
+
const DEFAULT_VARIANTS = [
|
|
204
|
+
{ name: 'Variant A', conversionMultiplier: 0.7, ttcMultiplier: 1.5, weight: 1 },
|
|
205
|
+
{ name: 'Variant B', conversionMultiplier: 1.3, ttcMultiplier: 0.7, weight: 1 },
|
|
206
|
+
{ name: 'Control', conversionMultiplier: 1.0, ttcMultiplier: 1.0, weight: 1 },
|
|
207
|
+
];
|
|
208
|
+
for (const f of funnels) {
|
|
209
|
+
if (!f || !f.experiment) continue;
|
|
210
|
+
const raw = f.experiment === true ? {} : f.experiment;
|
|
211
|
+
const name = raw.name || (f.name ? f.name + ' Experiment' : 'Unnamed Experiment');
|
|
212
|
+
const variants = (raw.variants && raw.variants.length)
|
|
213
|
+
? raw.variants.map(v => ({
|
|
214
|
+
name: v.name || 'Unnamed Variant',
|
|
215
|
+
conversionMultiplier: Number.isFinite(v.conversionMultiplier) ? Math.max(0.01, v.conversionMultiplier) : 1.0,
|
|
216
|
+
ttcMultiplier: Number.isFinite(v.ttcMultiplier) ? Math.max(0.01, v.ttcMultiplier) : 1.0,
|
|
217
|
+
weight: Number.isFinite(v.weight) && v.weight > 0 ? v.weight : 1,
|
|
218
|
+
}))
|
|
219
|
+
: DEFAULT_VARIANTS;
|
|
220
|
+
const startDays = Number.isFinite(raw.startDaysBeforeEnd) && raw.startDaysBeforeEnd > 0
|
|
221
|
+
? raw.startDaysBeforeEnd : 0;
|
|
222
|
+
const startUnix = startDays > 0 ? datasetEndUnix - startDays * 86400 : null;
|
|
223
|
+
f._experiment = { name, variants, startUnix };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Resolve `avgDevicePerUser` per Section 3.3 rules. Returns the integer device count
|
|
229
|
+
* the engine should use (0 = no device_id stamping at all, 1+ = pool size).
|
|
230
|
+
*
|
|
231
|
+
* @param {Partial<Dungeon>} config
|
|
232
|
+
* @returns {number}
|
|
233
|
+
*/
|
|
234
|
+
function resolveDevicesPerUser(config) {
|
|
235
|
+
const raw = config.avgDevicePerUser;
|
|
236
|
+
const hasAnon = config.hasAnonIds === true;
|
|
237
|
+
if (raw === undefined || raw === null) {
|
|
238
|
+
return hasAnon ? 1 : 0;
|
|
239
|
+
}
|
|
240
|
+
if (!Number.isFinite(raw)) {
|
|
241
|
+
return hasAnon ? 1 : 0;
|
|
242
|
+
}
|
|
243
|
+
const n = Math.round(raw);
|
|
244
|
+
if (n <= 0) return hasAnon ? 1 : 0;
|
|
245
|
+
return n;
|
|
246
|
+
}
|
|
247
|
+
|
|
148
248
|
/**
|
|
149
249
|
* Validates and enriches a dungeon configuration object
|
|
150
250
|
* @param {Partial<Dungeon>} config - Raw configuration object
|
|
@@ -153,6 +253,9 @@ function inferFunnels(events) {
|
|
|
153
253
|
export function validateDungeonConfig(config) {
|
|
154
254
|
const chance = u.getChance();
|
|
155
255
|
|
|
256
|
+
// Phase 1 — strip killed config keys before anything else reads them.
|
|
257
|
+
stripKilledConfigKeys(config);
|
|
258
|
+
|
|
156
259
|
// Transform SCD props to regular props if credentials are missing
|
|
157
260
|
// This MUST happen BEFORE we extract values from the config
|
|
158
261
|
transformSCDPropsWithoutCredentials(config);
|
|
@@ -243,7 +346,7 @@ export function validateDungeonConfig(config) {
|
|
|
243
346
|
// ── Resolve dataset window ──
|
|
244
347
|
// Preferred path: explicit datasetStart + datasetEnd → pinned, deterministic window.
|
|
245
348
|
// Fallback: numDays only → today_start - numDays back (sliding, warn-emitted).
|
|
246
|
-
const windowResolution = resolveDatasetWindow(config.datasetStart, config.datasetEnd, config.numDays);
|
|
349
|
+
const windowResolution = resolveDatasetWindow(config.datasetStart, config.datasetEnd, config.numDays, verbose);
|
|
247
350
|
const datasetStartUnix = windowResolution.datasetStartUnix;
|
|
248
351
|
const datasetEndUnix = windowResolution.datasetEndUnix;
|
|
249
352
|
numDays = windowResolution.numDays;
|
|
@@ -275,7 +378,7 @@ export function validateDungeonConfig(config) {
|
|
|
275
378
|
// only avgEventsPerUserPerDay would never trigger auto-batch.
|
|
276
379
|
if (numEvents >= 2_000_000 && config.batchSize === undefined) {
|
|
277
380
|
batchSize = 1_000_000;
|
|
278
|
-
console.warn(`⚠️ Auto-enabling batch mode: numEvents (${numEvents.toLocaleString()}) >= 2M. Using batchSize of ${batchSize.toLocaleString()}.`);
|
|
381
|
+
if (verbose) console.warn(`⚠️ Auto-enabling batch mode: numEvents (${numEvents.toLocaleString()}) >= 2M. Using batchSize of ${batchSize.toLocaleString()}.`);
|
|
279
382
|
}
|
|
280
383
|
|
|
281
384
|
// Resolve soup presets (intra-week / intra-day shape — must happen after numDays is computed)
|
|
@@ -319,7 +422,7 @@ export function validateDungeonConfig(config) {
|
|
|
319
422
|
throw new Error('Hook string did not evaluate to a function');
|
|
320
423
|
}
|
|
321
424
|
} catch (error) {
|
|
322
|
-
if (
|
|
425
|
+
if (verbose) {
|
|
323
426
|
console.warn(`\u26a0\ufe0f Failed to convert hook string to function: ${error.message}`);
|
|
324
427
|
console.warn('Using default pass-through hook');
|
|
325
428
|
}
|
|
@@ -329,7 +432,7 @@ export function validateDungeonConfig(config) {
|
|
|
329
432
|
|
|
330
433
|
// Ensure hook is a function
|
|
331
434
|
if (typeof hook !== 'function') {
|
|
332
|
-
if (
|
|
435
|
+
if (verbose) console.warn('\u26a0\ufe0f Hook is not a function, using default pass-through hook');
|
|
333
436
|
hook = (record) => record;
|
|
334
437
|
}
|
|
335
438
|
|
|
@@ -436,35 +539,31 @@ export function validateDungeonConfig(config) {
|
|
|
436
539
|
dataQuality = validateDataQuality(dataQuality);
|
|
437
540
|
}
|
|
438
541
|
|
|
439
|
-
//
|
|
440
|
-
|
|
441
|
-
if (subscription) {
|
|
442
|
-
subscription = validateSubscription(subscription);
|
|
443
|
-
}
|
|
542
|
+
// Phase 1: validate Funnel.attempts on every funnel (additive — most have none).
|
|
543
|
+
validateAttempts(funnels);
|
|
444
544
|
|
|
445
|
-
//
|
|
446
|
-
|
|
447
|
-
if (attribution) {
|
|
448
|
-
attribution = validateAttribution(attribution, numDays);
|
|
449
|
-
}
|
|
545
|
+
// Normalize experiment configs: true → default 3-variant, object → validated.
|
|
546
|
+
normalizeExperiments(funnels, datasetEndUnix);
|
|
450
547
|
|
|
451
|
-
//
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
548
|
+
// Phase 1: resolve multi-device config. `avgDevicePerUser` is the canonical knob;
|
|
549
|
+
// `hasAnonIds: true` aliases to 1. Default 0 = no device_id stamping (legacy).
|
|
550
|
+
const avgDevicePerUser = resolveDevicesPerUser(config);
|
|
551
|
+
// Keep hasAnonIds in sync — downstream code (utils.generateUser) still reads it.
|
|
552
|
+
// Setting it true here when avgDevicePerUser >= 1 lets the legacy device-pool
|
|
553
|
+
// generation in `person()` continue to allocate `anonymousIds[]` for the user.
|
|
554
|
+
const hasAnonIdsResolved = avgDevicePerUser >= 1;
|
|
456
555
|
|
|
457
|
-
//
|
|
458
|
-
|
|
459
|
-
if (
|
|
460
|
-
|
|
556
|
+
// Warn if isAuthEvent is set but avgDevicePerUser=0 — pre-auth device_only
|
|
557
|
+
// stamping degrades to user_id via the floor guard, defeating the identity model.
|
|
558
|
+
if (verbose && avgDevicePerUser === 0 && validatedEvents.some(e => e.isAuthEvent)) {
|
|
559
|
+
console.warn(
|
|
560
|
+
`⚠️ isAuthEvent requires avgDevicePerUser >= 1 to produce pre-auth anonymous events. ` +
|
|
561
|
+
`Set avgDevicePerUser or hasAnonIds: true.`
|
|
562
|
+
);
|
|
461
563
|
}
|
|
462
564
|
|
|
463
|
-
//
|
|
464
|
-
|
|
465
|
-
if (anomalies) {
|
|
466
|
-
anomalies = resolveAnomalies(anomalies, datasetStartUnix);
|
|
467
|
-
}
|
|
565
|
+
// Precompute whether any event has isAttributionEvent for UTM stamping logic.
|
|
566
|
+
const hasAttributionFlags = validatedEvents.some(e => e.isAttributionEvent);
|
|
468
567
|
|
|
469
568
|
// Build final config object
|
|
470
569
|
const validatedConfig = {
|
|
@@ -489,7 +588,8 @@ export function validateDungeonConfig(config) {
|
|
|
489
588
|
groupKeys,
|
|
490
589
|
groupProps,
|
|
491
590
|
lookupTables,
|
|
492
|
-
hasAnonIds,
|
|
591
|
+
hasAnonIds: hasAnonIdsResolved,
|
|
592
|
+
avgDevicePerUser,
|
|
493
593
|
hasSessionIds,
|
|
494
594
|
sessionTimeout: (typeof sessionTimeout === 'number' && sessionTimeout > 0) ? sessionTimeout : 30,
|
|
495
595
|
format,
|
|
@@ -501,6 +601,7 @@ export function validateDungeonConfig(config) {
|
|
|
501
601
|
hook,
|
|
502
602
|
hasAdSpend,
|
|
503
603
|
hasCampaigns,
|
|
604
|
+
hasAttributionFlags,
|
|
504
605
|
hasLocation,
|
|
505
606
|
hasAvatar,
|
|
506
607
|
isAnonymous,
|
|
@@ -515,16 +616,17 @@ export function validateDungeonConfig(config) {
|
|
|
515
616
|
bornRecentBias,
|
|
516
617
|
percentUsersBornInDataset,
|
|
517
618
|
preExistingSpread,
|
|
518
|
-
// Advanced features
|
|
619
|
+
// Advanced features (kept after 1.4)
|
|
519
620
|
personas,
|
|
520
621
|
worldEvents,
|
|
521
622
|
engagementDecay,
|
|
522
623
|
dataQuality,
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
624
|
+
// Killed in 1.4 — set to null so hooks/external code that reads these get a falsy value.
|
|
625
|
+
subscription: null,
|
|
626
|
+
attribution: null,
|
|
627
|
+
geo: null,
|
|
628
|
+
features: null,
|
|
629
|
+
anomalies: null
|
|
528
630
|
};
|
|
529
631
|
|
|
530
632
|
return validatedConfig;
|
|
@@ -689,131 +791,8 @@ function validateDataQuality(dq) {
|
|
|
689
791
|
return dq;
|
|
690
792
|
}
|
|
691
793
|
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
* @returns {import('../../types').Subscription}
|
|
696
|
-
*/
|
|
697
|
-
function validateSubscription(sub) {
|
|
698
|
-
if (!sub.plans || !Array.isArray(sub.plans) || sub.plans.length === 0) {
|
|
699
|
-
throw new Error('subscription.plans must be a non-empty array');
|
|
700
|
-
}
|
|
701
|
-
const hasDefault = sub.plans.some(p => p.default);
|
|
702
|
-
if (!hasDefault) sub.plans[0].default = true;
|
|
703
|
-
if (!sub.lifecycle) sub.lifecycle = {};
|
|
704
|
-
const lc = sub.lifecycle;
|
|
705
|
-
if (lc.trialToPayRate === undefined) lc.trialToPayRate = 0.3;
|
|
706
|
-
if (lc.upgradeRate === undefined) lc.upgradeRate = 0.1;
|
|
707
|
-
if (lc.downgradeRate === undefined) lc.downgradeRate = 0.03;
|
|
708
|
-
if (lc.churnRate === undefined) lc.churnRate = 0.05;
|
|
709
|
-
if (lc.winBackRate === undefined) lc.winBackRate = 0.1;
|
|
710
|
-
if (lc.winBackDelay === undefined) lc.winBackDelay = 30;
|
|
711
|
-
if (lc.paymentFailureRate === undefined) lc.paymentFailureRate = 0.02;
|
|
712
|
-
if (!sub.events) sub.events = {};
|
|
713
|
-
const ev = sub.events;
|
|
714
|
-
if (!ev.trialStarted) ev.trialStarted = "trial started";
|
|
715
|
-
if (!ev.subscribed) ev.subscribed = "subscription started";
|
|
716
|
-
if (!ev.upgraded) ev.upgraded = "plan upgraded";
|
|
717
|
-
if (!ev.downgraded) ev.downgraded = "plan downgraded";
|
|
718
|
-
if (!ev.renewed) ev.renewed = "subscription renewed";
|
|
719
|
-
if (!ev.cancelled) ev.cancelled = "subscription cancelled";
|
|
720
|
-
if (!ev.paymentFailed) ev.paymentFailed = "payment failed";
|
|
721
|
-
if (!ev.wonBack) ev.wonBack = "subscription reactivated";
|
|
722
|
-
return sub;
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
/**
|
|
726
|
-
* Validates attribution config
|
|
727
|
-
* @param {import('../../types').Attribution} attr
|
|
728
|
-
* @param {number} numDays
|
|
729
|
-
* @returns {import('../../types').Attribution}
|
|
730
|
-
*/
|
|
731
|
-
function validateAttribution(attr, numDays) {
|
|
732
|
-
if (!attr.campaigns || !Array.isArray(attr.campaigns)) {
|
|
733
|
-
throw new Error('attribution.campaigns must be an array');
|
|
734
|
-
}
|
|
735
|
-
if (attr.model === undefined) attr.model = "last_touch";
|
|
736
|
-
if (attr.window === undefined) attr.window = 7;
|
|
737
|
-
if (attr.organicRate === undefined) attr.organicRate = 0.4;
|
|
738
|
-
for (const c of attr.campaigns) {
|
|
739
|
-
if (!c.name) throw new Error('Each attribution campaign must have a name');
|
|
740
|
-
if (!c.source) throw new Error(`Attribution campaign "${c.name}" must have a source`);
|
|
741
|
-
if (!c.activeDays) throw new Error(`Attribution campaign "${c.name}" must have activeDays [start, end]`);
|
|
742
|
-
if (!c.dailyBudget) c.dailyBudget = [50, 200];
|
|
743
|
-
if (c.acquisitionRate === undefined) c.acquisitionRate = 0.02;
|
|
744
|
-
}
|
|
745
|
-
return attr;
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
/**
|
|
749
|
-
* Validates geo config
|
|
750
|
-
* @param {import('../../types').GeoConfig} geo
|
|
751
|
-
* @returns {import('../../types').GeoConfig}
|
|
752
|
-
*/
|
|
753
|
-
function validateGeo(geo) {
|
|
754
|
-
if (geo.sticky === undefined) geo.sticky = false;
|
|
755
|
-
if (geo.regions && !Array.isArray(geo.regions)) throw new Error('geo.regions must be an array');
|
|
756
|
-
if (geo.regionalLaunches && !Array.isArray(geo.regionalLaunches)) throw new Error('geo.regionalLaunches must be an array');
|
|
757
|
-
return geo;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
/**
|
|
761
|
-
* Resolves feature configs with logistic curve parameters
|
|
762
|
-
* @param {import('../../types').FeatureConfig[]} features
|
|
763
|
-
* @param {number} numDays
|
|
764
|
-
* @returns {import('../../types').FeatureConfig[]}
|
|
765
|
-
*/
|
|
766
|
-
function resolveFeatures(features, numDays) {
|
|
767
|
-
if (!Array.isArray(features) || features.length === 0) return null;
|
|
768
|
-
const curvePresets = {
|
|
769
|
-
fast: { k: 0.3, midpoint: 7 },
|
|
770
|
-
slow: { k: 0.08, midpoint: 30 },
|
|
771
|
-
instant: { k: 10, midpoint: 0 }
|
|
772
|
-
};
|
|
773
|
-
return features.map(f => {
|
|
774
|
-
if (!f.name) throw new Error('Each feature must have a name');
|
|
775
|
-
if (f.launchDay === undefined) throw new Error(`Feature "${f.name}" must have a launchDay`);
|
|
776
|
-
if (!f.property) throw new Error(`Feature "${f.name}" must have a property`);
|
|
777
|
-
if (!f.values || !Array.isArray(f.values) || f.values.length === 0) {
|
|
778
|
-
throw new Error(`Feature "${f.name}" must have a non-empty values array`);
|
|
779
|
-
}
|
|
780
|
-
if (!f.affectsEvents) f.affectsEvents = "*";
|
|
781
|
-
if (!f.adoptionCurve) f.adoptionCurve = "slow";
|
|
782
|
-
if (typeof f.adoptionCurve === 'string') {
|
|
783
|
-
f._resolvedCurve = curvePresets[f.adoptionCurve] || curvePresets.slow;
|
|
784
|
-
} else {
|
|
785
|
-
f._resolvedCurve = f.adoptionCurve;
|
|
786
|
-
}
|
|
787
|
-
// Pre-compute adopted values to avoid array allocation in hot loop
|
|
788
|
-
f._adoptedValues = f.values.length > 1 ? f.values.slice(1) : f.values;
|
|
789
|
-
return f;
|
|
790
|
-
});
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
/**
|
|
794
|
-
* Resolves anomaly configs with absolute timestamps
|
|
795
|
-
* @param {import('../../types').AnomalyConfig[]} anomalies
|
|
796
|
-
* @param {number} beginUnix - Dataset start (unix seconds)
|
|
797
|
-
* @returns {import('../../types').AnomalyConfig[]}
|
|
798
|
-
*/
|
|
799
|
-
function resolveAnomalies(anomalies, beginUnix) {
|
|
800
|
-
if (!Array.isArray(anomalies) || anomalies.length === 0) return null;
|
|
801
|
-
return anomalies.map(a => {
|
|
802
|
-
if (!a.type) throw new Error('Each anomaly must have a type');
|
|
803
|
-
if (!a.event) throw new Error('Each anomaly must have an event name');
|
|
804
|
-
const resolved = { ...a };
|
|
805
|
-
if (a.day !== undefined) {
|
|
806
|
-
resolved._startUnix = beginUnix + (a.day * 86400);
|
|
807
|
-
if (a.duration) {
|
|
808
|
-
resolved._endUnix = resolved._startUnix + (a.duration * 86400);
|
|
809
|
-
} else if (a.window) {
|
|
810
|
-
resolved._endUnix = resolved._startUnix + (a.window * 86400);
|
|
811
|
-
} else {
|
|
812
|
-
resolved._endUnix = resolved._startUnix + 86400; // default 1 day
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
return resolved;
|
|
816
|
-
});
|
|
817
|
-
}
|
|
794
|
+
// validateSubscription / validateAttribution / validateGeo / resolveFeatures /
|
|
795
|
+
// resolveAnomalies were removed in 1.4 along with their respective config keys.
|
|
796
|
+
// `stripKilledConfigKeys` deletes the inputs before they reach the validator body.
|
|
818
797
|
|
|
819
798
|
export { inferFunnels, transformSCDPropsWithoutCredentials };
|
package/lib/core/storage.js
CHANGED
|
@@ -50,6 +50,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
50
50
|
let writeDir;
|
|
51
51
|
let isBatchMode = runtime.isBatchMode || false;
|
|
52
52
|
let isWriting = false; // Prevent concurrent writes
|
|
53
|
+
const writtenFiles = [];
|
|
53
54
|
|
|
54
55
|
// Determine write directory
|
|
55
56
|
const dataFolder = path.resolve("./data");
|
|
@@ -175,6 +176,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
175
176
|
|
|
176
177
|
// Write to disk/cloud - always blocking to prevent OOM
|
|
177
178
|
const writeResult = await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
|
|
179
|
+
writtenFiles.push(writePath);
|
|
178
180
|
return writeResult;
|
|
179
181
|
} finally {
|
|
180
182
|
isWriting = false; // Release the lock
|
|
@@ -231,6 +233,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
231
233
|
const dataToWrite = [...arr];
|
|
232
234
|
arr.length = 0; // Clear array after copying data
|
|
233
235
|
await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
|
|
236
|
+
writtenFiles.push(writePath);
|
|
234
237
|
// Data now lives on disk, not in arr — mirror what transformThenPush
|
|
235
238
|
// does when crossing BATCH_SIZE so the Mixpanel sender knows to read
|
|
236
239
|
// from disk instead of from the (now empty) in-memory array.
|
|
@@ -249,6 +252,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
249
252
|
enrichedArray.flush = flush;
|
|
250
253
|
enrichedArray.getWriteDir = getWriteDir;
|
|
251
254
|
enrichedArray.getWritePath = getWritePath;
|
|
255
|
+
enrichedArray.getWrittenFiles = () => [...writtenFiles];
|
|
252
256
|
|
|
253
257
|
// Add additional properties from rest
|
|
254
258
|
for (const key in rest) {
|
|
@@ -384,7 +388,7 @@ export class StorageManager {
|
|
|
384
388
|
const batchSize = config.batchSize || 1_000_000;
|
|
385
389
|
const numEvents = config.numEvents || 0;
|
|
386
390
|
|
|
387
|
-
if (batchSize < numEvents) {
|
|
391
|
+
if (batchSize < numEvents && config.verbose) {
|
|
388
392
|
console.warn(
|
|
389
393
|
`⚠️ writeToDisk is false but batchSize (${batchSize.toLocaleString()}) < numEvents (${numEvents.toLocaleString()}). ` +
|
|
390
394
|
`Batch files will be written to disk temporarily to avoid OOM. ` +
|
package/lib/generators/events.js
CHANGED
|
@@ -10,25 +10,28 @@
|
|
|
10
10
|
/** @typedef {import('../../types').Context} Context */
|
|
11
11
|
|
|
12
12
|
import dayjs from "dayjs";
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
13
14
|
import * as u from "../utils/utils.js";
|
|
14
15
|
import { dataLogger as logger } from "../utils/logger.js";
|
|
15
16
|
|
|
16
17
|
// Keys that must never be nulled by data quality gremlins
|
|
17
18
|
const NULL_EXEMPT_KEYS = new Set(['event', 'time', 'insert_id', 'user_id', 'device_id', 'distinct_id', '_drop', '_anomaly', '_persona']);
|
|
18
19
|
|
|
20
|
+
|
|
19
21
|
/**
|
|
20
22
|
* Creates a Mixpanel event with a flat shape
|
|
21
|
-
* @param {Context} context
|
|
22
|
-
* @param {string} distinct_id
|
|
23
|
+
* @param {Context} context
|
|
24
|
+
* @param {string} distinct_id
|
|
23
25
|
* @param {number} earliestTime - Unix timestamp for earliest possible event time
|
|
24
26
|
* @param {Object} chosenEvent - Event configuration object
|
|
25
27
|
* @param {string[]} [anonymousIds] - Array of anonymous/device IDs
|
|
26
|
-
* @param {string[]} [sessionIds] - Array of session IDs
|
|
27
28
|
* @param {Object} [superProps] - Super properties to add to event
|
|
28
29
|
* @param {Array} [groupKeys] - Group key configurations
|
|
29
|
-
* @param {boolean} [isFirstEvent]
|
|
30
|
-
* @param {boolean} [skipDefaults]
|
|
31
|
-
* @
|
|
30
|
+
* @param {boolean} [isFirstEvent]
|
|
31
|
+
* @param {boolean} [skipDefaults]
|
|
32
|
+
* @param {Object} [featureCtx] - Feature context (persona, worldEvents, dataQuality)
|
|
33
|
+
* @param {Object} [identityCtx] - Phase 2 identity context ({ stamping, devicePool })
|
|
34
|
+
* @returns {Promise<Object>}
|
|
32
35
|
*/
|
|
33
36
|
export async function makeEvent(
|
|
34
37
|
context,
|
|
@@ -36,12 +39,12 @@ export async function makeEvent(
|
|
|
36
39
|
earliestTime,
|
|
37
40
|
chosenEvent,
|
|
38
41
|
anonymousIds = [],
|
|
39
|
-
sessionIds = [],
|
|
40
42
|
superProps = {},
|
|
41
43
|
groupKeys = [],
|
|
42
44
|
isFirstEvent = false,
|
|
43
45
|
skipDefaults = false,
|
|
44
|
-
featureCtx = {}
|
|
46
|
+
featureCtx = {},
|
|
47
|
+
identityCtx = null
|
|
45
48
|
) {
|
|
46
49
|
// Validate required parameters
|
|
47
50
|
if (!distinct_id) throw new Error("no distinct_id");
|
|
@@ -89,9 +92,16 @@ export async function makeEvent(
|
|
|
89
92
|
defaultProps.browser = u.choose(defaults.browsers());
|
|
90
93
|
}
|
|
91
94
|
|
|
92
|
-
// Add campaigns with attribution likelihood
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
// Add campaigns with attribution likelihood.
|
|
96
|
+
// When any event has isAttributionEvent, only stamp UTMs on those events (25% chance).
|
|
97
|
+
// Otherwise, backwards-compat: ~25% of all events get UTMs.
|
|
98
|
+
if (hasCampaigns) {
|
|
99
|
+
const shouldStamp = config.hasAttributionFlags
|
|
100
|
+
? (chosenEvent.isAttributionEvent && chance.bool({ likelihood: 25 }))
|
|
101
|
+
: chance.bool({ likelihood: 25 });
|
|
102
|
+
if (shouldStamp) {
|
|
103
|
+
defaultProps.campaigns = u.pickRandom(defaults.campaigns());
|
|
104
|
+
}
|
|
95
105
|
}
|
|
96
106
|
|
|
97
107
|
// PERFORMANCE: Use pre-computed device pool instead of rebuilding every time
|
|
@@ -110,19 +120,32 @@ export async function makeEvent(
|
|
|
110
120
|
eventTemplate.time = dayjs.unix(unixTime).toISOString();
|
|
111
121
|
}
|
|
112
122
|
|
|
113
|
-
//
|
|
114
|
-
|
|
115
|
-
|
|
123
|
+
// ── Phase 2 identity stamping ──
|
|
124
|
+
// `identityCtx.stamping` modes:
|
|
125
|
+
// "both" → user_id + device_id (when pool non-empty). DEFAULT.
|
|
126
|
+
// "user_only" → user_id only (post-auth funnel events).
|
|
127
|
+
// "device_only" → device_id only (pre-auth funnel events).
|
|
128
|
+
// "stitch" → both (the one stitch event per converted born-in-dataset user).
|
|
129
|
+
// Callers (funnels.js / user-loop.js) compute the mode based on the user's auth state
|
|
130
|
+
// and the funnel's `isAuthEvent` placement. Default "both" preserves backwards-compat
|
|
131
|
+
// for dungeons that don't flag auth events. The legacy 42%-per-event user_id dice is
|
|
132
|
+
// gone — every event now gets user_id by default.
|
|
133
|
+
const stamping = (identityCtx && identityCtx.stamping) || 'both';
|
|
134
|
+
const wantsUser = stamping === 'both' || stamping === 'user_only' || stamping === 'stitch';
|
|
135
|
+
const wantsDevice = stamping === 'both' || stamping === 'device_only' || stamping === 'stitch';
|
|
136
|
+
const devicePool = (identityCtx && identityCtx.devicePool) || anonymousIds || [];
|
|
137
|
+
|
|
138
|
+
if (wantsDevice && devicePool && devicePool.length) {
|
|
139
|
+
eventTemplate.device_id = u.pickRandom(devicePool);
|
|
116
140
|
}
|
|
117
|
-
|
|
118
|
-
// Session IDs are assigned post-hoc in user-loop.js based on temporal gaps
|
|
119
|
-
|
|
120
|
-
// Sometimes add user_id (for attribution modeling)
|
|
121
|
-
if (!isFirstEvent && chance.bool({ likelihood: 42 })) {
|
|
141
|
+
if (wantsUser) {
|
|
122
142
|
eventTemplate.user_id = distinct_id;
|
|
123
143
|
}
|
|
144
|
+
// Session IDs are assigned post-hoc in user-loop.js based on temporal gaps
|
|
124
145
|
|
|
125
|
-
//
|
|
146
|
+
// Floor: every event must carry at least one of user_id / device_id (storage layer
|
|
147
|
+
// rejects events lacking both). If the stamping mode is "device_only" but there's no
|
|
148
|
+
// device pool, fall back to user_id rather than producing an invalid event.
|
|
126
149
|
if (!eventTemplate.user_id && !eventTemplate.device_id) {
|
|
127
150
|
eventTemplate.user_id = distinct_id;
|
|
128
151
|
}
|
|
@@ -165,36 +188,13 @@ export async function makeEvent(
|
|
|
165
188
|
addGroupProperties(eventTemplate, groupKeys);
|
|
166
189
|
|
|
167
190
|
// ── Event-level features (applied before hooks, so hooks can override) ──
|
|
168
|
-
const { userLocation, persona, worldEventsTimeline,
|
|
169
|
-
|
|
170
|
-
// Feature 6: Attribution — stamp UTM properties on events as touchpoints
|
|
171
|
-
// Mixpanel attribution analysis needs UTM on EVENTS, not just profiles.
|
|
172
|
-
// Pattern: first events carry acquisition UTM, later events occasionally carry re-engagement UTM.
|
|
173
|
-
if (userCampaign) {
|
|
174
|
-
// ~40% of events carry UTM (simulates page loads, session starts, ad clicks)
|
|
175
|
-
// First events (isFirstEvent) always carry UTM (acquisition touchpoint)
|
|
176
|
-
if (isFirstEvent || chance.bool({ likelihood: 40 })) {
|
|
177
|
-
eventTemplate.utm_source = userCampaign.source;
|
|
178
|
-
eventTemplate.utm_campaign = userCampaign.name;
|
|
179
|
-
if (userCampaign.medium) eventTemplate.utm_medium = userCampaign.medium;
|
|
180
|
-
if (userCampaign.utm_content) eventTemplate.utm_content = userCampaign.utm_content;
|
|
181
|
-
if (userCampaign.utm_term) eventTemplate.utm_term = userCampaign.utm_term;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// Feature 7: Sticky geo location
|
|
186
|
-
if (userLocation && geoConfig?.sticky) {
|
|
187
|
-
// Override the random location with user's sticky location
|
|
188
|
-
for (const key in userLocation) {
|
|
189
|
-
eventTemplate[key] = userLocation[key];
|
|
190
|
-
}
|
|
191
|
-
}
|
|
191
|
+
const { userLocation, persona, worldEventsTimeline, dataQuality: dq } = featureCtx;
|
|
192
192
|
|
|
193
|
-
// Perf 1: Compute eventUnix once for all time-based checks. World events
|
|
194
|
-
//
|
|
195
|
-
//
|
|
193
|
+
// Perf 1: Compute eventUnix once for all time-based checks. World events
|
|
194
|
+
// were resolved against the dataset window (no shift), and event times now
|
|
195
|
+
// also live in that same window — direct unix conversion works.
|
|
196
196
|
let eventUnix = null;
|
|
197
|
-
if (
|
|
197
|
+
if (worldEventsTimeline && eventTemplate.time) {
|
|
198
198
|
eventUnix = dayjs(eventTemplate.time).unix();
|
|
199
199
|
}
|
|
200
200
|
|
|
@@ -222,47 +222,6 @@ export async function makeEvent(
|
|
|
222
222
|
}
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
-
// Feature 8: Progressive feature adoption
|
|
226
|
-
if (resolvedFeatures && eventUnix !== null) {
|
|
227
|
-
const daysSinceBegin = (eventUnix - context.FIXED_BEGIN) / 86400;
|
|
228
|
-
for (const feat of resolvedFeatures) {
|
|
229
|
-
const affects = feat.affectsEvents;
|
|
230
|
-
if (affects !== "*" && !(Array.isArray(affects) && affects.includes(eventTemplate.event))) continue;
|
|
231
|
-
if (daysSinceBegin < feat.launchDay) {
|
|
232
|
-
if (feat.defaultBefore !== undefined) {
|
|
233
|
-
eventTemplate[feat.property] = feat.defaultBefore;
|
|
234
|
-
}
|
|
235
|
-
} else {
|
|
236
|
-
const daysSinceLaunch = daysSinceBegin - feat.launchDay;
|
|
237
|
-
const { k, midpoint } = feat._resolvedCurve || { k: 0.08, midpoint: 30 };
|
|
238
|
-
const adoptionProb = 1 / (1 + Math.exp(-k * (daysSinceLaunch - midpoint)));
|
|
239
|
-
if (chance.bool({ likelihood: Math.min(100, adoptionProb * 100) })) {
|
|
240
|
-
eventTemplate[feat.property] = u.pickRandom(feat._adoptedValues);
|
|
241
|
-
} else if (feat.defaultBefore !== undefined) {
|
|
242
|
-
eventTemplate[feat.property] = feat.defaultBefore;
|
|
243
|
-
} else if (feat.values.length > 0) {
|
|
244
|
-
eventTemplate[feat.property] = feat.values[0];
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// Feature 9: Anomaly extreme values
|
|
251
|
-
if (resolvedAnomalies && eventTemplate.event) {
|
|
252
|
-
for (const a of resolvedAnomalies) {
|
|
253
|
-
if (a.type === 'extreme_value' && a.event === eventTemplate.event && a.property) {
|
|
254
|
-
if (chance.bool({ likelihood: (a.frequency ?? 0.001) * 100 })) {
|
|
255
|
-
const currentVal = eventTemplate[a.property];
|
|
256
|
-
if (typeof currentVal === 'number') {
|
|
257
|
-
eventTemplate[a.property] = currentVal * (a.multiplier || 10);
|
|
258
|
-
}
|
|
259
|
-
if (a.tag) eventTemplate._anomaly = a.tag;
|
|
260
|
-
if (a.properties) Object.assign(eventTemplate, a.properties);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
|
|
266
225
|
// Feature 4: Data quality — null injection and timezone confusion
|
|
267
226
|
if (dq) {
|
|
268
227
|
// Null injection
|
|
@@ -287,10 +246,7 @@ export async function makeEvent(
|
|
|
287
246
|
}
|
|
288
247
|
}
|
|
289
248
|
|
|
290
|
-
|
|
291
|
-
const distinctId = eventTemplate.user_id || eventTemplate.device_id || eventTemplate.distinct_id || distinct_id;
|
|
292
|
-
const tuple = `${eventTemplate.event}-${eventTemplate.time}-${distinctId}`;
|
|
293
|
-
eventTemplate.insert_id = u.quickHash(tuple);
|
|
249
|
+
eventTemplate.insert_id = randomUUID();
|
|
294
250
|
|
|
295
251
|
// Call hook if configured (hooks override everything — they are the final authority)
|
|
296
252
|
const { hook } = config;
|