@ak--47/dungeon-master 1.6.4 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/HOOKS.md +49 -0
- package/README.md +129 -8
- package/docs/guides/1.7.0-upgrade-guide.md +154 -0
- package/index.js +16 -1
- package/lib/core/config-validator.js +243 -13
- package/lib/core/context.js +39 -0
- package/lib/core/dungeon-loader.js +4 -1
- package/lib/generators/events.js +53 -7
- package/lib/generators/funnels.js +85 -11
- package/lib/generators/profiles.js +9 -4
- package/lib/orchestrators/mixpanel-sender.js +13 -1
- package/lib/orchestrators/user-loop.js +239 -9
- package/lib/utils/conditions.js +62 -0
- package/lib/utils/json-evaluator.js +12 -2
- package/lib/utils/utils.js +78 -7
- package/lib/verify/schema-validator.js +8 -0
- package/package.json +2 -10
- package/types.d.ts +303 -34
|
@@ -92,12 +92,18 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
92
92
|
if (hashVal < cumWeight) { chosenVariant = expCfg.variants[vi]; chosenIdx = vi; break; }
|
|
93
93
|
}
|
|
94
94
|
experimentVariant = chosenVariant.name;
|
|
95
|
-
funnel.conversionRate =
|
|
96
|
-
Math.round((funnel.conversionRate || 50) * chosenVariant.conversionMultiplier))
|
|
95
|
+
funnel.conversionRate = saturateConversionRate(context, funnel,
|
|
96
|
+
Math.round((funnel.conversionRate || 50) * chosenVariant.conversionMultiplier), `experiment "${experimentName}" variant "${experimentVariant}"`, 1);
|
|
97
97
|
funnel.timeToConvert = Math.max(0.1,
|
|
98
98
|
(funnel.timeToConvert || 1) * chosenVariant.ttcMultiplier);
|
|
99
99
|
funnel._experimentName = experimentName;
|
|
100
100
|
funnel._experimentVariant = experimentVariant;
|
|
101
|
+
// v1.7.0 (P0-2): record the assignment so the user loop can stamp
|
|
102
|
+
// `Experiment: <name>` on the profile. Only when the bucketing is sticky
|
|
103
|
+
// (a re-rolled variant has no single per-user value) and stampProfile is on.
|
|
104
|
+
if (expCfg.sticky !== false && expCfg.stampProfile !== false && featureCtx && featureCtx.experimentAssignments) {
|
|
105
|
+
featureCtx.experimentAssignments.set(experimentName, experimentVariant);
|
|
106
|
+
}
|
|
101
107
|
funnel.sequence = ["$experiment_started", ...funnel.sequence];
|
|
102
108
|
experimentMeta = {
|
|
103
109
|
name: experimentName,
|
|
@@ -113,7 +119,13 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
113
119
|
// Apply persona and world-event modifiers to the funnel BEFORE the hook fires,
|
|
114
120
|
// so funnel-pre sees the effective rate and has final authority.
|
|
115
121
|
if (persona && persona.conversionModifier) {
|
|
116
|
-
funnel.conversionRate =
|
|
122
|
+
funnel.conversionRate = saturateConversionRate(context, funnel,
|
|
123
|
+
Math.round((funnel.conversionRate || 50) * persona.conversionModifier), `persona "${persona.name}" conversionModifier`, 0);
|
|
124
|
+
}
|
|
125
|
+
// v1.7.0 (P1-3): persona time-to-convert multiplier. Composes after the
|
|
126
|
+
// experiment ttcMultiplier, before the funnel-pre hook (hook keeps final say).
|
|
127
|
+
if (persona && Number.isFinite(persona.ttcModifier) && persona.ttcModifier !== 1) {
|
|
128
|
+
funnel.timeToConvert = Math.max(0.1, (funnel.timeToConvert || 1) * persona.ttcModifier);
|
|
117
129
|
}
|
|
118
130
|
const resolvedWorldEvents = /** @type {import('../../types').ResolvedWorldEvent[]} */ (config.worldEvents);
|
|
119
131
|
if (resolvedWorldEvents && firstEventTime) {
|
|
@@ -122,7 +134,8 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
122
134
|
const seq = funnel.sequence || [];
|
|
123
135
|
const affects = we.affectsEvents;
|
|
124
136
|
if (affects === "*" || (Array.isArray(affects) && seq.some(s => affects.includes(s)))) {
|
|
125
|
-
funnel.conversionRate =
|
|
137
|
+
funnel.conversionRate = saturateConversionRate(context, funnel,
|
|
138
|
+
Math.round((funnel.conversionRate || 50) * we.conversionModifier), `worldEvent "${we.name}" conversionModifier`, 0);
|
|
126
139
|
}
|
|
127
140
|
}
|
|
128
141
|
}
|
|
@@ -144,6 +157,16 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
144
157
|
experiment: experimentMeta,
|
|
145
158
|
});
|
|
146
159
|
|
|
160
|
+
// v1.7.0 (P2-4): a funnel-pre hook that leaves conversionRate above 100 is
|
|
161
|
+
// clamped downstream (processEventRepeats) — record the saturation so the
|
|
162
|
+
// author learns the asked-for lift did not land. The engine can only see its
|
|
163
|
+
// own clamp: a hook's own `Math.min(95, rate * 3)` never reaches here.
|
|
164
|
+
// Record only — the value itself is left for processEventRepeats to clamp exactly
|
|
165
|
+
// as before, so pre-1.7 output is unchanged.
|
|
166
|
+
if (Number.isFinite(funnel.conversionRate) && funnel.conversionRate > 100) {
|
|
167
|
+
saturateConversionRate(context, funnel, funnel.conversionRate, 'funnel-pre hook', 0);
|
|
168
|
+
}
|
|
169
|
+
|
|
147
170
|
// Extract funnel configuration (post-hook — hook's mutations are the final word)
|
|
148
171
|
let {
|
|
149
172
|
sequence,
|
|
@@ -160,18 +183,21 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
160
183
|
const { distinct_id, created, anonymousIds = [] } = user;
|
|
161
184
|
const { superProps = {}, groupKeys = [] } = config;
|
|
162
185
|
|
|
186
|
+
// v1.7.0 (P1-1): value context for funnel-level and step-level property thunks.
|
|
187
|
+
const valueCtx = { profile, config, time: Number.isFinite(firstEventTime) ? firstEventTime * 1000 : undefined };
|
|
188
|
+
|
|
163
189
|
// Choose properties for this funnel instance
|
|
164
190
|
const chosenFunnelProps = { ...props, ...superProps };
|
|
165
191
|
for (const key in props) {
|
|
166
192
|
try {
|
|
167
|
-
chosenFunnelProps[key] = u.choose(chosenFunnelProps[key]);
|
|
193
|
+
chosenFunnelProps[key] = u.choose(chosenFunnelProps[key], valueCtx);
|
|
168
194
|
} catch (e) {
|
|
169
195
|
logger.error({ err: e, key, funnel: funnel.sequence.join(" > ") }, `Error processing property ${key} in funnel`);
|
|
170
196
|
}
|
|
171
197
|
}
|
|
172
198
|
|
|
173
199
|
// Build event specifications for funnel steps
|
|
174
|
-
const funnelPossibleEvents = buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, expName, expVariant);
|
|
200
|
+
const funnelPossibleEvents = buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, expName, expVariant, valueCtx);
|
|
175
201
|
|
|
176
202
|
// Handle repeat logic and conversion rate adjustment
|
|
177
203
|
let { processedEvents, adjustedConversionRate } = processEventRepeats(
|
|
@@ -298,6 +324,9 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
298
324
|
persona: featureCtx.persona || persona || null,
|
|
299
325
|
userCampaign: featureCtx.userCampaign || null,
|
|
300
326
|
userLocation: featureCtx.userLocation || null,
|
|
327
|
+
// v1.7.0: value context + sticky event props flow through to makeEvent.
|
|
328
|
+
profile: featureCtx.profile || profile || null,
|
|
329
|
+
stickyValues: featureCtx.stickyValues || null,
|
|
301
330
|
worldEventsTimeline: featureCtx.worldEventsTimeline || context.config.worldEvents || null,
|
|
302
331
|
dataQuality: featureCtx.dataQuality || context.config.dataQuality || null,
|
|
303
332
|
latestTime: Number.isFinite(featureCtx.latestTime) ? featureCtx.latestTime : undefined,
|
|
@@ -399,7 +428,7 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
399
428
|
// Resolve declared props on the exclusion event's config.
|
|
400
429
|
if (excConfig && excConfig.properties) {
|
|
401
430
|
for (const k of Object.keys(excConfig.properties)) {
|
|
402
|
-
try { cloned[k] = u.choose(excConfig.properties[k]); }
|
|
431
|
+
try { cloned[k] = u.choose(excConfig.properties[k], { ...valueCtx, event: cloned, time: Date.parse(cloned.time) }); }
|
|
403
432
|
catch (e) { cloned[k] = null; }
|
|
404
433
|
}
|
|
405
434
|
}
|
|
@@ -429,6 +458,33 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
429
458
|
return [finalEvents, doesUserConvert, authTimeMs];
|
|
430
459
|
}
|
|
431
460
|
|
|
461
|
+
/**
|
|
462
|
+
* v1.7.0 (P2-4): clamp a modified conversionRate into `[floor, 100]` and, when the
|
|
463
|
+
* requested value exceeded 100, record ONE aggregated warning per funnel+source on
|
|
464
|
+
* the context (never per user — this runs in the hot loop).
|
|
465
|
+
*
|
|
466
|
+
* @param {Context} context
|
|
467
|
+
* @param {Object} funnel
|
|
468
|
+
* @param {number} requested - pre-clamp rate
|
|
469
|
+
* @param {string} source - what produced the rate (persona / experiment / world event / hook)
|
|
470
|
+
* @param {number} floor - lower bound (experiments floor at 1, others at 0)
|
|
471
|
+
* @returns {number} clamped rate
|
|
472
|
+
*/
|
|
473
|
+
function saturateConversionRate(context, funnel, requested, source, floor) {
|
|
474
|
+
const applied = Math.min(100, Math.max(floor, requested));
|
|
475
|
+
if (requested > 100 && context && typeof context.addWarning === 'function') {
|
|
476
|
+
const label = funnel.name || (Array.isArray(funnel.sequence) ? funnel.sequence.filter(s => s !== '$experiment_started').join(' > ') : '?');
|
|
477
|
+
context.addWarning({
|
|
478
|
+
key: `funnels[${label}].conversionRate:${source}`,
|
|
479
|
+
requested,
|
|
480
|
+
applied: 100,
|
|
481
|
+
reason: `${source} pushed conversionRate above 100; saturated at 100, so the intended lift did not fully land. Lower the base conversionRate or the multiplier.`,
|
|
482
|
+
severity: 'clamp',
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
return applied;
|
|
486
|
+
}
|
|
487
|
+
|
|
432
488
|
/**
|
|
433
489
|
* Builds event specifications for funnel steps
|
|
434
490
|
* @param {Context} context - Context object
|
|
@@ -437,9 +493,10 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
|
|
|
437
493
|
* @param {number} bindPropsIndex - Index at which to bind properties (if applicable)
|
|
438
494
|
* @param {string} [experimentName] - Name of experiment (if experiment is enabled)
|
|
439
495
|
* @param {string} [experimentVariant] - Variant name (A, B, or C)
|
|
496
|
+
* @param {Object} [valueCtx] - v1.7.0 value context for property thunks
|
|
440
497
|
* @returns {Array} Array of event specifications
|
|
441
498
|
*/
|
|
442
|
-
function buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, experimentName, experimentVariant) {
|
|
499
|
+
function buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, experimentName, experimentVariant, valueCtx = undefined) {
|
|
443
500
|
const { config } = context;
|
|
444
501
|
|
|
445
502
|
return sequence.map((eventName, currentIndex) => {
|
|
@@ -463,10 +520,14 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex,
|
|
|
463
520
|
properties: { ...foundEvent.properties }
|
|
464
521
|
} : { event: eventName, properties: {} };
|
|
465
522
|
|
|
466
|
-
// Process event properties
|
|
523
|
+
// Process event properties.
|
|
524
|
+
// v1.7.0 (P1-1): context-aware thunks (`(ctx) => …`) are DEFERRED — left in
|
|
525
|
+
// place so makeEvent resolves them per step with the real `ctx.event` and
|
|
526
|
+
// `ctx.time`. Everything else resolves here exactly as before.
|
|
467
527
|
for (const key in eventSpec.properties) {
|
|
528
|
+
if (u.isContextAware(eventSpec.properties[key])) continue;
|
|
468
529
|
try {
|
|
469
|
-
eventSpec.properties[key] = u.choose(eventSpec.properties[key]);
|
|
530
|
+
eventSpec.properties[key] = u.choose(eventSpec.properties[key], valueCtx);
|
|
470
531
|
} catch (e) {
|
|
471
532
|
logger.error({ err: e, key, event: eventSpec.event }, `Error processing property ${key} in ${eventSpec.event} event`);
|
|
472
533
|
}
|
|
@@ -636,9 +697,22 @@ async function generateFunnelEvents(
|
|
|
636
697
|
const stampingByIndex = (identityArgs && identityArgs.stampingByIndex) || null;
|
|
637
698
|
const devicePool = (identityArgs && identityArgs.devicePool) || null;
|
|
638
699
|
|
|
700
|
+
// v1.7.0: step 0 reports its TimeSoup time through a synchronous side channel.
|
|
701
|
+
// Each map callback runs synchronously up to makeEvent's first `await` (the
|
|
702
|
+
// event hook), which is AFTER the time is set — so by the time callback i>0
|
|
703
|
+
// starts, `funnelStartMs` is known and the step's final time can be pinned
|
|
704
|
+
// before its properties resolve. RNG order is unchanged.
|
|
705
|
+
let funnelStartMs = null;
|
|
706
|
+
|
|
639
707
|
const finalEvents = await Promise.all(eventsWithTiming.map(async (event, index) => {
|
|
640
708
|
const stamping = stampingByIndex ? stampingByIndex[index] : 'both';
|
|
641
709
|
const identityCtx = (devicePool || stamping !== 'both') ? { stamping, devicePool } : null;
|
|
710
|
+
let stepFeatureCtx = featureCtx;
|
|
711
|
+
if (index === 0) {
|
|
712
|
+
stepFeatureCtx = { ...featureCtx, onTimeResolved: (ms) => { funnelStartMs = ms; } };
|
|
713
|
+
} else if (funnelStartMs !== null && Number.isFinite(event.relativeTimeMs)) {
|
|
714
|
+
stepFeatureCtx = { ...featureCtx, fixedTimeMs: funnelStartMs + event.relativeTimeMs };
|
|
715
|
+
}
|
|
642
716
|
const newEvent = await makeEvent(
|
|
643
717
|
context,
|
|
644
718
|
distinct_id,
|
|
@@ -649,7 +723,7 @@ async function generateFunnelEvents(
|
|
|
649
723
|
groupKeys,
|
|
650
724
|
false,
|
|
651
725
|
false,
|
|
652
|
-
|
|
726
|
+
stepFeatureCtx,
|
|
653
727
|
identityCtx
|
|
654
728
|
);
|
|
655
729
|
|
|
@@ -18,19 +18,24 @@ import { dataLogger as logger } from "../utils/logger.js";
|
|
|
18
18
|
export async function makeProfile(context, props = {}, defaults = {}) {
|
|
19
19
|
// Update operation counter
|
|
20
20
|
context.incrementOperations();
|
|
21
|
-
|
|
21
|
+
|
|
22
22
|
// Keys that should not be processed with the choose function
|
|
23
23
|
const keysToNotChoose = ["anonymousIds", "sessionIds"];
|
|
24
24
|
|
|
25
25
|
// Start with defaults
|
|
26
26
|
const profile = { ...defaults };
|
|
27
27
|
|
|
28
|
+
// v1.7.0 (P1-1): value functions see the profile as it is being built.
|
|
29
|
+
// Keys resolve in insertion order, so a later key can read an earlier one
|
|
30
|
+
// (`revenue: (ctx) => ctx.profile.plan === 'pro' ? 100 : 10`).
|
|
31
|
+
const valueCtx = { profile, config: context.config };
|
|
32
|
+
|
|
28
33
|
// Process default values first
|
|
29
34
|
for (const key in profile) {
|
|
30
35
|
if (keysToNotChoose.includes(key)) continue;
|
|
31
|
-
|
|
36
|
+
|
|
32
37
|
try {
|
|
33
|
-
profile[key] = u.choose(profile[key]);
|
|
38
|
+
profile[key] = u.choose(profile[key], valueCtx);
|
|
34
39
|
} catch (e) {
|
|
35
40
|
logger.error({ err: e, key }, `Error processing default property ${key}`);
|
|
36
41
|
// Keep original value on error
|
|
@@ -40,7 +45,7 @@ export async function makeProfile(context, props = {}, defaults = {}) {
|
|
|
40
45
|
// Process provided props (these override defaults)
|
|
41
46
|
for (const key in props) {
|
|
42
47
|
try {
|
|
43
|
-
profile[key] = u.choose(props[key]);
|
|
48
|
+
profile[key] = u.choose(props[key], valueCtx);
|
|
44
49
|
} catch (e) {
|
|
45
50
|
logger.error({ err: e, key }, `Error processing property ${key}`);
|
|
46
51
|
// Keep original value on error
|
|
@@ -100,6 +100,7 @@ async function _sendToMixpanel(context) {
|
|
|
100
100
|
const commonOpts = {
|
|
101
101
|
region,
|
|
102
102
|
fixData: true,
|
|
103
|
+
matchMixpanelDefaults: true,
|
|
103
104
|
v2_compat: true,
|
|
104
105
|
verbose: false,
|
|
105
106
|
forceStream: true,
|
|
@@ -174,7 +175,18 @@ async function _sendToMixpanel(context) {
|
|
|
174
175
|
progressCallback: makeProgressCallback(userTotal),
|
|
175
176
|
});
|
|
176
177
|
log(` -> ${comma(imported.success)} user profiles sent\n`);
|
|
177
|
-
|
|
178
|
+
// v1.7.0 (R2-5): make the receipt self-explanatory. `generated` counts every
|
|
179
|
+
// profile the engine pushed to storage (bots included); `dropped_anonymous`
|
|
180
|
+
// counts the `_drop`-flagged anonymous non-converters that never reach
|
|
181
|
+
// /engage. `generated - dropped_anonymous - failed === success` is checkable.
|
|
182
|
+
// Counters live on the context and tick at push time, so they hold in
|
|
183
|
+
// batch mode where the in-memory array has been flushed.
|
|
184
|
+
const generated = context.runtime.profilesGenerated;
|
|
185
|
+
const dropped_anonymous = context.runtime.profilesDropped;
|
|
186
|
+
importResults.users = { ...imported, generated, dropped_anonymous };
|
|
187
|
+
if (imported.success + (imported.failed || 0) + dropped_anonymous !== generated) {
|
|
188
|
+
log(` !! profile receipt does not reconcile: ${comma(generated)} generated - ${comma(dropped_anonymous)} dropped anonymous - ${comma(imported.failed || 0)} failed != ${comma(imported.success)} success\n`);
|
|
189
|
+
}
|
|
178
190
|
}
|
|
179
191
|
|
|
180
192
|
// Import ad spend data
|
|
@@ -16,6 +16,11 @@ import { makeFunnel } from "../generators/funnels.js";
|
|
|
16
16
|
import { makeUserProfile } from "../generators/profiles.js";
|
|
17
17
|
import { makeSCD } from "../generators/scd.js";
|
|
18
18
|
import { buildCurveWeightFn, expectedActiveDays } from "../utils/retention-curve.js";
|
|
19
|
+
import { matchConditions } from "../utils/conditions.js";
|
|
20
|
+
|
|
21
|
+
// v1.7.0 (P0-1): matching moved to lib/utils/conditions.js (operator maps).
|
|
22
|
+
// Re-exported here so existing imports keep working.
|
|
23
|
+
export { matchConditions };
|
|
19
24
|
|
|
20
25
|
/**
|
|
21
26
|
* Main user generation loop that creates users, their profiles, events, and SCDs
|
|
@@ -69,6 +74,23 @@ export async function userLoop(context) {
|
|
|
69
74
|
// Track if we've already logged the strict event count message
|
|
70
75
|
let hasLoggedStrictCountReached = false;
|
|
71
76
|
|
|
77
|
+
// v1.7.0 (R2-3): strictEventCount bookkeeping. `strictBudgetSum` is the sum
|
|
78
|
+
// of per-user budgets handed out so far; with the STORED count it yields the
|
|
79
|
+
// realized events-per-budget ratio, which the controller below uses to scale
|
|
80
|
+
// the remaining users' budgets so the run lands on `numEvents` and the final
|
|
81
|
+
// trim makes it exact. Before 1.7.0 the flag stopped on the GENERATED count
|
|
82
|
+
// (which includes `_drop`'d events) and never topped up, so it delivered ~6%
|
|
83
|
+
// short with 4x headroom available.
|
|
84
|
+
let strictBudgetSum = 0;
|
|
85
|
+
let strictUsersBudgeted = 0;
|
|
86
|
+
|
|
87
|
+
// v1.7.0 run-level counters for aggregated warnings (P0-1 item 5, P1-5).
|
|
88
|
+
const anyConditionedFunnel = Array.isArray(funnels) && funnels.some(f => f && f.conditions);
|
|
89
|
+
let usersMatchingNoFunnel = 0;
|
|
90
|
+
let usersChurned = 0;
|
|
91
|
+
const personaMultipliersInPlay = Array.isArray(personas) && personas.some(p => p && Number.isFinite(p.eventMultiplier) && p.eventMultiplier !== 1);
|
|
92
|
+
const campaignKeys = ['utm_source', 'utm_campaign', 'utm_medium', 'utm_content', 'utm_term'];
|
|
93
|
+
|
|
72
94
|
// Handle graceful shutdown on SIGINT (Ctrl+C).
|
|
73
95
|
// CRITICAL: listener MUST be removed in a `finally` block — pre-fix, throws /
|
|
74
96
|
// cancellation paths leaked listeners across test runs, accumulating until
|
|
@@ -87,8 +109,9 @@ export async function userLoop(context) {
|
|
|
87
109
|
// Bail out if cancelled
|
|
88
110
|
if (cancelled) return;
|
|
89
111
|
|
|
90
|
-
// Bail out early if strictEventCount is enabled and we've hit numEvents
|
|
91
|
-
|
|
112
|
+
// Bail out early if strictEventCount is enabled and we've hit numEvents.
|
|
113
|
+
// v1.7.0: compares the STORED count (post-drop), not the generated count.
|
|
114
|
+
if (strictEventCount && context.getStoredEventCount() >= numEvents) {
|
|
92
115
|
if (verbose && !hasLoggedStrictCountReached) {
|
|
93
116
|
console.log(`\n\u2713 Reached target of ${numEvents.toLocaleString()} events with strict event count enabled. Stopping user generation.`);
|
|
94
117
|
hasLoggedStrictCountReached = true;
|
|
@@ -213,17 +236,40 @@ export async function userLoop(context) {
|
|
|
213
236
|
// Feature 1: Merge persona properties into profile (before hook, so hook can override)
|
|
214
237
|
if (persona && persona.properties) {
|
|
215
238
|
for (const [key, value] of Object.entries(persona.properties)) {
|
|
216
|
-
profile[key] = u.choose(value);
|
|
239
|
+
profile[key] = u.choose(value, { profile, config });
|
|
217
240
|
}
|
|
218
241
|
profile._persona = persona.name;
|
|
219
242
|
}
|
|
220
243
|
|
|
244
|
+
// v1.7.0 (P1-4): one acquisition campaign per user. Draw a campaign
|
|
245
|
+
// template at birth and stamp its UTM keys on the profile. Any UTM key the
|
|
246
|
+
// profile already carries (persona `properties`) wins over the draw, so
|
|
247
|
+
// "the paid-search persona" is `properties: { utm_source: 'google' }`. The
|
|
248
|
+
// `user` hook runs after this and can override too; the touchpoint pass
|
|
249
|
+
// reads the FINAL profile values. Opt-in: no RNG draw unless enabled.
|
|
250
|
+
if (config.campaignPerUser && config.hasCampaigns) {
|
|
251
|
+
const template = u.pickRandom(defaults.campaigns());
|
|
252
|
+
if (template && typeof template === 'object') {
|
|
253
|
+
for (const key of campaignKeys) {
|
|
254
|
+
if (profile[key] !== undefined) continue;
|
|
255
|
+
if (template[key] === undefined) continue;
|
|
256
|
+
profile[key] = u.choose(template[key], { profile, config });
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
221
261
|
// Build feature context for event generation
|
|
222
262
|
const featureCtx = {
|
|
223
263
|
persona,
|
|
224
264
|
userLocation,
|
|
225
265
|
worldEventsTimeline: worldEvents,
|
|
226
266
|
dataQuality,
|
|
267
|
+
// v1.7.0: value context (P1-1), sticky event props (P1-2), per-user
|
|
268
|
+
// campaign (P1-4), experiment assignments collector (P0-2).
|
|
269
|
+
profile,
|
|
270
|
+
stickyValues: null,
|
|
271
|
+
userCampaign: null,
|
|
272
|
+
experimentAssignments: new Map(),
|
|
227
273
|
};
|
|
228
274
|
|
|
229
275
|
// Call user hook after profile creation (hooks override persona properties).
|
|
@@ -246,6 +292,32 @@ export async function userLoop(context) {
|
|
|
246
292
|
}
|
|
247
293
|
}
|
|
248
294
|
|
|
295
|
+
// v1.7.0 (P1-2): resolve sticky event props ONCE per user, after the user
|
|
296
|
+
// hook so hook overrides are honored. Keys declared in userProps / persona
|
|
297
|
+
// properties copy from the profile; keys declared only in superProps are
|
|
298
|
+
// resolved here once (instead of per event) and held constant.
|
|
299
|
+
if (Array.isArray(config.stickyEventProps) && config.stickyEventProps.length) {
|
|
300
|
+
const superOnly = new Set(config._stickySuperOnly || []);
|
|
301
|
+
const stickyValues = {};
|
|
302
|
+
for (const key of config.stickyEventProps) {
|
|
303
|
+
stickyValues[key] = superOnly.has(key)
|
|
304
|
+
? u.choose(config.superProps[key], { profile, config })
|
|
305
|
+
: profile[key];
|
|
306
|
+
}
|
|
307
|
+
featureCtx.stickyValues = stickyValues;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// v1.7.0 (P1-4): the user's campaign, read from the FINAL profile so persona
|
|
311
|
+
// properties and the user hook both flow into the touchpoint pass.
|
|
312
|
+
if (config.campaignPerUser && config.hasCampaigns) {
|
|
313
|
+
const userCampaign = {};
|
|
314
|
+
let any = false;
|
|
315
|
+
for (const key of campaignKeys) {
|
|
316
|
+
if (profile[key] !== undefined && profile[key] !== null) { userCampaign[key] = profile[key]; any = true; }
|
|
317
|
+
}
|
|
318
|
+
featureCtx.userCampaign = any ? userCampaign : null;
|
|
319
|
+
}
|
|
320
|
+
|
|
249
321
|
// SCD creation
|
|
250
322
|
// @ts-ignore
|
|
251
323
|
const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
|
|
@@ -305,6 +377,31 @@ export async function userLoop(context) {
|
|
|
305
377
|
}
|
|
306
378
|
numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
|
|
307
379
|
|
|
380
|
+
// v1.7.0 (R2-3): strictEventCount budget controller. Born-in-dataset users
|
|
381
|
+
// carry a pro-rated budget and `_drop`s thin the rest, so the raw budgets
|
|
382
|
+
// sum below `numEvents` by design (~6% measured). Scale this user's budget
|
|
383
|
+
// by (events still needed) / (events the remaining users would deliver at
|
|
384
|
+
// the realized yield), aiming 2% high so the final per-user trim has
|
|
385
|
+
// something to trim. Clamped to [0.25, 4] so no single user is distorted
|
|
386
|
+
// into a whale. Only fires under the flag — the default path is untouched.
|
|
387
|
+
if (strictEventCount && numEventsThisUserWillPreform > 0) {
|
|
388
|
+
const delivered = context.getStoredEventCount();
|
|
389
|
+
const remainingTarget = numEvents - delivered;
|
|
390
|
+
const remainingUsers = Math.max(1, numUsers - i);
|
|
391
|
+
const meanBudget = strictUsersBudgeted > 0 ? strictBudgetSum / strictUsersBudgeted : numEventsThisUserWillPreform;
|
|
392
|
+
const yieldRatio = strictBudgetSum > 0 ? delivered / strictBudgetSum : 1;
|
|
393
|
+
const expectedRemaining = meanBudget * yieldRatio * remainingUsers;
|
|
394
|
+
if (remainingTarget > 0 && expectedRemaining > 0) {
|
|
395
|
+
// Overshoot grows as the tail shrinks (last user aims 2x its share) so a
|
|
396
|
+
// low-yield final user cannot leave the run short; the trim removes excess.
|
|
397
|
+
const overshoot = 1.02 + 1 / remainingUsers;
|
|
398
|
+
const scale = Math.max(0.25, Math.min(4, (remainingTarget * overshoot) / expectedRemaining));
|
|
399
|
+
numEventsThisUserWillPreform = Math.max(1, Math.round(numEventsThisUserWillPreform * scale));
|
|
400
|
+
}
|
|
401
|
+
strictBudgetSum += numEventsThisUserWillPreform;
|
|
402
|
+
strictUsersBudgeted++;
|
|
403
|
+
}
|
|
404
|
+
|
|
308
405
|
let userFirstEventTime;
|
|
309
406
|
|
|
310
407
|
// ── v1.5 Active-day scheduling ──
|
|
@@ -354,6 +451,12 @@ export async function userLoop(context) {
|
|
|
354
451
|
const usageFunnels = funnels.filter((f) => !f.isFirstFunnel)
|
|
355
452
|
.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
|
|
356
453
|
.reduce(weighFunnels, []);
|
|
454
|
+
// v1.7.0 (P0-1 item 5): with conditions in play, a user who matches no
|
|
455
|
+
// funnel at all silently falls through to standalone events. Counted here,
|
|
456
|
+
// reported once per run (see end of loop).
|
|
457
|
+
if (anyConditionedFunnel && firstFunnels.length === 0 && !usageFunnels.some(f => !f._catchAll)) {
|
|
458
|
+
usersMatchingNoFunnel++;
|
|
459
|
+
}
|
|
357
460
|
|
|
358
461
|
const secondsInDay = 86400;
|
|
359
462
|
const noise = () => chance.integer({ min: 0, max: secondsInDay });
|
|
@@ -627,6 +730,24 @@ export async function userLoop(context) {
|
|
|
627
730
|
return Number.isFinite(t) ? t <= userChurnTimeMs : true;
|
|
628
731
|
});
|
|
629
732
|
}
|
|
733
|
+
if (userChurned) usersChurned++;
|
|
734
|
+
|
|
735
|
+
// v1.7.0 (P0-3): world-event amplification. `volumeMultiplier > 1` clones
|
|
736
|
+
// affected in-window events (fresh insert_id, spread uniformly across the
|
|
737
|
+
// window) so daily volume reaches the stated multiple. Runs after the churn
|
|
738
|
+
// cut (no post-churn clones) and before decay / hooks, which see the clones.
|
|
739
|
+
const resolvedWorldEvents = /** @type {import('../../types').ResolvedWorldEvent[] | null} */ (worldEvents);
|
|
740
|
+
if (resolvedWorldEvents && usersEvents.length > 0 && worldEventsAmplify(resolvedWorldEvents)) {
|
|
741
|
+
usersEvents = amplifyWorldEvents(usersEvents, resolvedWorldEvents, chance, context.FIXED_NOW);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// v1.7.0 (P0-2): stamp sticky experiment assignments on the profile as
|
|
745
|
+
// `Experiment: <name>` before the everything hook, so hooks see it too.
|
|
746
|
+
if (featureCtx.experimentAssignments.size > 0) {
|
|
747
|
+
for (const [expName, variantName] of featureCtx.experimentAssignments) {
|
|
748
|
+
profile[`Experiment: ${expName}`] = variantName;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
630
751
|
|
|
631
752
|
// Feature 3: Engagement decay — filter behavioral events
|
|
632
753
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
@@ -696,7 +817,7 @@ export async function userLoop(context) {
|
|
|
696
817
|
// preserves realistic touch shape — last-10-window attribution then
|
|
697
818
|
// gives meaningful first/last-touch results.
|
|
698
819
|
if (config.hasCampaigns && usersEvents.length > 0) {
|
|
699
|
-
applyTouchpointCap(usersEvents, config, defaults, chance);
|
|
820
|
+
applyTouchpointCap(usersEvents, config, defaults, chance, featureCtx.userCampaign);
|
|
700
821
|
}
|
|
701
822
|
|
|
702
823
|
// v1.5.1: anonymous non-converters never call $identify in production,
|
|
@@ -819,9 +940,25 @@ export async function userLoop(context) {
|
|
|
819
940
|
u.assignSessionIds(sortedView, sessionTimeout);
|
|
820
941
|
}
|
|
821
942
|
|
|
943
|
+
// v1.7.0 (R2-3): strict mode never exceeds numEvents. Trim this user's
|
|
944
|
+
// stream to the remaining room with a seeded uniform sample (keeps the
|
|
945
|
+
// time shape; preserves array order). Only under the flag.
|
|
946
|
+
if (strictEventCount && usersEvents.length > 0) {
|
|
947
|
+
const room = numEvents - context.getStoredEventCount();
|
|
948
|
+
if (room <= 0) {
|
|
949
|
+
usersEvents = [];
|
|
950
|
+
} else if (usersEvents.length > room) {
|
|
951
|
+
const keep = new Set(chance.pickset(usersEvents, room));
|
|
952
|
+
usersEvents = usersEvents.filter(e => keep.has(e));
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
822
956
|
// Store all user data (skip profile push when a hook returned null
|
|
823
957
|
// for type='user' — see dropUserProfile above).
|
|
824
958
|
if (!dropUserProfile) {
|
|
959
|
+
// v1.7.0 (R2-5): receipt counters tick at push time (batch-mode safe).
|
|
960
|
+
context.incrementProfilesGenerated();
|
|
961
|
+
if (profile._drop) context.incrementProfilesDropped();
|
|
825
962
|
await userProfilesData.hookPush(profile);
|
|
826
963
|
}
|
|
827
964
|
|
|
@@ -868,6 +1005,42 @@ export async function userLoop(context) {
|
|
|
868
1005
|
if (dataQuality && dataQuality.botUsers > 0) {
|
|
869
1006
|
await generateBotUsers(context, dataQuality, storage);
|
|
870
1007
|
}
|
|
1008
|
+
|
|
1009
|
+
// ── v1.7.0 aggregated run-level warnings (surfaced as result.warnings) ──
|
|
1010
|
+
if (typeof context.addWarning === 'function') {
|
|
1011
|
+
if (usersMatchingNoFunnel > 0) {
|
|
1012
|
+
context.addWarning({
|
|
1013
|
+
key: 'funnels.conditions',
|
|
1014
|
+
requested: numUsers,
|
|
1015
|
+
applied: numUsers - usersMatchingNoFunnel,
|
|
1016
|
+
reason: `${usersMatchingNoFunnel} of ${numUsers} users matched no funnel's conditions and generated standalone events only; check that the conditioned funnels cover every segment`,
|
|
1017
|
+
severity: 'warn',
|
|
1018
|
+
count: usersMatchingNoFunnel,
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
if (personaMultipliersInPlay && numUsers > 0 && usersChurned / numUsers > 0.5) {
|
|
1022
|
+
context.addWarning({
|
|
1023
|
+
key: 'personas.eventMultiplier',
|
|
1024
|
+
requested: usersChurned,
|
|
1025
|
+
applied: numUsers,
|
|
1026
|
+
reason: `${usersChurned} of ${numUsers} users were ended by an isChurnEvent; churn caps every user's lifetime event count, so personas[].eventMultiplier cannot lift a persona past it (measured 1.04x for an asked 3x). Lower the churn event's weight, raise returnLikelihood, or drive churn from a hook.`,
|
|
1027
|
+
severity: 'warn',
|
|
1028
|
+
count: usersChurned,
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
if (strictEventCount) {
|
|
1032
|
+
const stored = context.getStoredEventCount();
|
|
1033
|
+
if (stored < numEvents) {
|
|
1034
|
+
context.addWarning({
|
|
1035
|
+
key: 'numEvents',
|
|
1036
|
+
requested: numEvents,
|
|
1037
|
+
applied: stored,
|
|
1038
|
+
reason: `strictEventCount could not reach numEvents: the ${numUsers} users' capacity (rate × active days, minus drops) ran out ${numEvents - stored} events short. Raise numUsers, numDays, or avgEventsPerUserPerDay.`,
|
|
1039
|
+
severity: 'warn',
|
|
1040
|
+
});
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
871
1044
|
} finally {
|
|
872
1045
|
// Always remove the SIGINT listener — even if userLoop throws or is
|
|
873
1046
|
// cancelled. Pre-fix this leaked across test runs and stalled workers.
|
|
@@ -884,11 +1057,61 @@ export function weighFunnels(acc, funnel) {
|
|
|
884
1057
|
return acc;
|
|
885
1058
|
}
|
|
886
1059
|
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
1060
|
+
/**
|
|
1061
|
+
* v1.7.0 (P0-3): true when any resolved world event asks for amplification.
|
|
1062
|
+
* @param {import('../../types').ResolvedWorldEvent[]} worldEvents
|
|
1063
|
+
*/
|
|
1064
|
+
function worldEventsAmplify(worldEvents) {
|
|
1065
|
+
return worldEvents.some(we => we && ((we.volumeMultiplier > 1) || (we.aftermath && we.aftermath.volumeMultiplier > 1)));
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
/**
|
|
1069
|
+
* v1.7.0 (P0-3): clone affected events so volume inside a world-event window (or
|
|
1070
|
+
* its aftermath) reaches `volumeMultiplier`× the baseline.
|
|
1071
|
+
*
|
|
1072
|
+
* For every event that falls inside an amplifying window and matches
|
|
1073
|
+
* `affectsEvents`, add `floor(mult - 1)` clones plus one more with probability
|
|
1074
|
+
* `frac(mult)` — so `2.5` means one guaranteed clone and a 50% second. Each clone
|
|
1075
|
+
* gets a FRESH `insert_id` (Mixpanel dedupes on it — a spread that keeps the
|
|
1076
|
+
* source id is silently dropped at ingest) and a timestamp drawn uniformly across
|
|
1077
|
+
* the window, never past `fixedNow`, so the surge is spread over the window's
|
|
1078
|
+
* days rather than stacked one second after its source. Mutates nothing; returns
|
|
1079
|
+
* a new array. Deterministic under the seeded chance.
|
|
1080
|
+
*
|
|
1081
|
+
* @param {Object[]} events - the user's surviving events
|
|
1082
|
+
* @param {import('../../types').ResolvedWorldEvent[]} worldEvents
|
|
1083
|
+
* @param {Object} chance - seeded chance instance
|
|
1084
|
+
* @param {number} fixedNow - dataset end (unix seconds)
|
|
1085
|
+
* @returns {Object[]}
|
|
1086
|
+
*/
|
|
1087
|
+
export function amplifyWorldEvents(events, worldEvents, chance, fixedNow) {
|
|
1088
|
+
const clones = [];
|
|
1089
|
+
for (const ev of events) {
|
|
1090
|
+
if (!ev || !ev.time) continue;
|
|
1091
|
+
const evUnix = Math.floor(Date.parse(ev.time) / 1000);
|
|
1092
|
+
if (!Number.isFinite(evUnix)) continue;
|
|
1093
|
+
for (const we of worldEvents) {
|
|
1094
|
+
if (!we) continue;
|
|
1095
|
+
const inMain = evUnix >= we.startUnix && evUnix < we.endUnix;
|
|
1096
|
+
const inAftermath = !!we.aftermathEndUnix && evUnix >= we.endUnix && evUnix < we.aftermathEndUnix;
|
|
1097
|
+
if (!inMain && !inAftermath) continue;
|
|
1098
|
+
const affects = we.affectsEvents;
|
|
1099
|
+
if (!(affects === "*" || (Array.isArray(affects) && affects.includes(ev.event)))) continue;
|
|
1100
|
+
const mult = inMain ? we.volumeMultiplier : (we.aftermath && we.aftermath.volumeMultiplier) || 1;
|
|
1101
|
+
if (!(mult > 1)) continue;
|
|
1102
|
+
const windowStart = inMain ? we.startUnix : we.endUnix;
|
|
1103
|
+
const windowEnd = Math.min(inMain ? we.endUnix : we.aftermathEndUnix, fixedNow);
|
|
1104
|
+
if (!(windowEnd > windowStart)) continue;
|
|
1105
|
+
let copies = Math.floor(mult - 1);
|
|
1106
|
+
const frac = mult - 1 - copies;
|
|
1107
|
+
if (frac > 0 && chance.bool({ likelihood: frac * 100 })) copies++;
|
|
1108
|
+
for (let c = 0; c < copies; c++) {
|
|
1109
|
+
const t = chance.integer({ min: windowStart, max: windowEnd - 1 });
|
|
1110
|
+
clones.push({ ...ev, time: dayjs.unix(t).toISOString(), insert_id: randomUUID() });
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
890
1113
|
}
|
|
891
|
-
return
|
|
1114
|
+
return clones.length ? events.concat(clones) : events;
|
|
892
1115
|
}
|
|
893
1116
|
|
|
894
1117
|
// ── v1.5 Active-Day Plan Helpers ──
|
|
@@ -1058,8 +1281,11 @@ function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDa
|
|
|
1058
1281
|
* @param {Object} config - Validated dungeon config
|
|
1059
1282
|
* @param {Object} defaults - Context.defaults (provides campaigns())
|
|
1060
1283
|
* @param {Object} chance - Seeded chance instance
|
|
1284
|
+
* @param {Object|null} [userCampaign] - v1.7.0 (P1-4): the user's resolved UTM values
|
|
1285
|
+
* (from the profile) when `campaignPerUser` is on. Stamped verbatim on every
|
|
1286
|
+
* sampled touchpoint instead of re-drawing a template per event.
|
|
1061
1287
|
*/
|
|
1062
|
-
function applyTouchpointCap(events, config, defaults, chance) {
|
|
1288
|
+
function applyTouchpointCap(events, config, defaults, chance, userCampaign = null) {
|
|
1063
1289
|
const cap = Number.isFinite(config.maxTouchpointsPerUser)
|
|
1064
1290
|
? config.maxTouchpointsPerUser
|
|
1065
1291
|
: 10;
|
|
@@ -1101,6 +1327,10 @@ function applyTouchpointCap(events, config, defaults, chance) {
|
|
|
1101
1327
|
|
|
1102
1328
|
// Stamp UTMs on each sampled event using a campaign template.
|
|
1103
1329
|
for (const ev of sample) {
|
|
1330
|
+
if (userCampaign) {
|
|
1331
|
+
for (const [k, v] of Object.entries(userCampaign)) ev[k] = v;
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1104
1334
|
const campaignTemplate = u.pickRandom(defaults.campaigns());
|
|
1105
1335
|
if (!campaignTemplate || typeof campaignTemplate !== 'object') continue;
|
|
1106
1336
|
for (const [k, v] of Object.entries(campaignTemplate)) {
|