@ak--47/dungeon-master 1.5.0 → 1.5.2

package/lib/orchestrators/user-loop.js

@@ -15,6 +15,7 @@ import { makeEvent } from "../generators/events.js";
 import { makeFunnel } from "../generators/funnels.js";
 import { makeUserProfile } from "../generators/profiles.js";
 import { makeSCD } from "../generators/scd.js";
+import { buildCurveWeightFn, expectedActiveDays } from "../utils/retention-curve.js";
 
 /**
  * Main user generation loop that creates users, their profiles, events, and SCDs
@@ -43,6 +44,7 @@ export async function userLoop(context) {
 		numDays,
 		avgEventsPerUserPerDay,
 		avgActiveDaysPerUser,
+		retentionCurve,
 		percentUsersBornInDataset = 15,
 		strictEventCount = false,
 		bornRecentBias = 0, // -1..1; positive = births skew toward end of window
@@ -119,7 +121,12 @@ export async function userLoop(context) {
 				percentComplete: Math.min(100, Math.round((context.getUserCount() / numUsers) * 100))
 			});
 
-			const userId = chance.guid();
+			// v1.5.1: distinct_id sourced from a separate `userChance` when
+			// `Dungeon.userSeed` is set. This lets sharded runs (e.g., kodiak
+			// Cloud Run Job) generate the SAME user pool with DIFFERENT events
+			// across shards. Falls back to the event chance when userSeed is
+			// unset so existing dungeons stay byte-identical.
+			const userId = u.getUserChance().guid();
 			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
 			const { distinct_id, created } = user;
 			const userDevicePool = (user.anonymousIds && user.anonymousIds.length) ? user.anonymousIds.slice() : null;
@@ -219,9 +226,14 @@ export async function userLoop(context) {
 				dataQuality,
 			};
 
-			// Call user hook after profile creation (hooks override persona properties)
+			// Call user hook after profile creation (hooks override persona properties).
+			// v1.5.1: an explicit `null` return is interpreted as "drop the user
+			// PROFILE record" — the user's events still generate normally. Used
+			// by sharded runs that have one canonical chunk per bucket emit profiles
+			// while every other chunk skips them (see dungeons/user/kodiak/).
+			let dropUserProfile = false;
 			if (config.hook) {
-				await config.hook(profile, "user", {
+				const hookedProfile = await config.hook(profile, "user", {
 					user,
 					config,
 					userIsBornInDataset,
@@ -229,6 +241,9 @@ export async function userLoop(context) {
 					datasetStart: context.DATASET_START_SECONDS,
 					datasetEnd: context.DATASET_END_SECONDS
 				});
+				if (hookedProfile === null) {
+					dropUserProfile = true;
+				}
 			}
 
 			// SCD creation
@@ -268,19 +283,25 @@ export async function userLoop(context) {
 				: numDays;
 			const userEventBudget = ratePerDay * userActiveDays;
 
-			let numEventsThisUserWillPreform = Math.floor(chance.normal({
+			// v1.5.1 (TODO #10): per-user event budget.
+			//
+			// Removed:
+			//   - the 0.714 magic dampening factor (pre-existing vestigial constant)
+			//   - the ×5 / ×0.333 "power user" / "low activity" dice rolls
+			//     (E[dice mult] ≈ 1.62, combined with 0.714 ≈ 1.16x inflation
+			//     by design — produced 60-100% systematic overshoot on
+			//     `numEvents` targets across all macros).
+			//
+			// New: pure normal distribution around `userEventBudget` with
+			// `dev = userEventBudget / 3` (~68% of users within ±1σ of target,
+			// ~95% within ±2σ). Real heavy-tail behavior should come from
+			// `personas` (`eventMultiplier`) — explicit, opt-in, documented.
+			let numEventsThisUserWillPreform = Math.max(0, Math.round(chance.normal({
 				mean: userEventBudget,
-				dev: userEventBudget / u.integer(u.integer(2, 5), u.integer(2, 7))
-			}) * 0.714159265359);
-
-			// Power users and low-activity users logic
+				dev: userEventBudget / 3,
+			})));
 			if (persona) {
-				// Persona-driven event multiplier replaces the old dice rolls
 				numEventsThisUserWillPreform *= persona.eventMultiplier;
-			} else {
-				// Legacy behavior when no personas configured
-				chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
-				chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
 			}
 			numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
 
@@ -300,12 +321,17 @@ export async function userLoop(context) {
 			// `{ plan, pickedDayBuckets }`. `pickedDayBuckets` flows into
 			// `applyEngagementDecay` so the decay filter never drops the last event
 			// on a picked day — preserving the configured distinct-day count.
-			const dayPlanResult = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser))
+			// v1.5.1: also fires when `retentionCurve` is set (curve wins over
+			// explicit `avgActiveDaysPerUser` — see TODO #4 resolution (b)).
+			const hasActiveDaysKnob = (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && Number.isFinite(avgActiveDaysPerUser));
+			const hasRetentionCurve = !!retentionCurve;
+			const dayPlanResult = (hasActiveDaysKnob || hasRetentionCurve)
 				? buildActiveDayPlan({
 					adjustedCreated, fixedBegin: context.FIXED_BEGIN, fixedNow: context.FIXED_NOW,
 					avgActiveDaysPerUser, userActiveDays,
 					numEvents: numEventsThisUserWillPreform,
 					dowWeights: soupCfgForActiveDay.dayOfWeekWeights,
+					retentionCurve,
 					chance,
 				})
 				: null;
@@ -422,9 +448,13 @@ export async function userLoop(context) {
 
 				userFirstEventTime = firstAttemptFirstEventTime !== null
 					? firstAttemptFirstEventTime
-					: adjustedCreated.subtract(noise(), 'seconds').unix();
+					: Math.max(adjustedCreated.subtract(noise(), 'seconds').unix(), context.FIXED_BEGIN);
 			} else {
-				userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+				// v1.5.1: clamp at FIXED_BEGIN so pre-existing-user events stay strictly
+				// inside the dataset window. Without this, `noise()` (up to 1 day) can
+				// shift the first-event time before FIXED_BEGIN — fine when the dataset
+				// is years long, but visible at sub-day chunk windows (kodiak shards).
+				userFirstEventTime = Math.max(adjustedCreated.subtract(noise(), 'seconds').unix(), context.FIXED_BEGIN);
 			}
 
 			// ALL SUBSEQUENT EVENTS (funnels for converted users, standalone for all)
@@ -458,6 +488,14 @@ export async function userLoop(context) {
 			// out cleanly rather than crashing on `pick([])`.
 			const hasUsageFunnels = usageFunnels.length > 0;
 			const hasStandaloneEvents = weightedEvents.length > 0;
+			// v1.5.1 (TODO #10): shortest usage-funnel length. Used to decide
+			// whether the remaining budget can fit another funnel run — funnels
+			// add N events at once, so iterations near the budget ceiling
+			// systematically overshot. Now we redirect to standalone (or break)
+			// when the remaining headroom is smaller than the smallest funnel.
+			const minUsageFunnelLen = hasUsageFunnels
+				? Math.min(...usageFunnels.map(f => Array.isArray(f.sequence) ? f.sequence.length : 1))
+				: Infinity;
 			// Hard ceiling on iterations: defends against pathological configs where
 			// every funnel produces 0 surviving events (e.g. ttc > numDays so step1
 			// uniform anchor always lands too late). Cap at 2× expected iteration count
@@ -483,7 +521,16 @@ export async function userLoop(context) {
 				// Nothing to generate: user converted via firstFunnel only and has no
 				// usage funnels and no standalone events. Stop attempting.
 				if ((!hasUsageFunnels || !userConverted) && !hasStandaloneEvents) break;
-				if (hasUsageFunnels && userConverted) {
+				// v1.5.1 (TODO #10): if the remaining budget can't fit even the
+				// shortest usage funnel, prefer a standalone event (or break if
+				// no standalone). Without this, the funnel branch added N events
+				// at once and overshot `numEvents` by `(funnel_length - 1)` per
+				// terminal iteration — measured ~5-15% systematic overshoot.
+				const remainingBudget = numEventsThisUserWillPreform - numEventsPreformed;
+				const useFunnel = hasUsageFunnels && userConverted && remainingBudget >= minUsageFunnelLen;
+				// Budget exhausted for funnels AND no standalone → break.
+				if (!useFunnel && !hasStandaloneEvents) break;
+				if (useFunnel) {
 					const currentFunnel = chance.pickone(usageFunnels);
 					const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
 					// Anchor cursor at picked day's start when active-day mode is on,
@@ -533,7 +580,12 @@ export async function userLoop(context) {
 					// `earliestTime`, which the now-deleted bunchIntoSessions used to paper
 					// over. With bunchIntoSessions removed, TimeSoup is the time source.
 					const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
-					const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
+					// v1.5.1: pass `config.superProps` so standalone events get super-property
+					// stamping. Pre-1.5.1, standalone events received `{}` here, but the
+					// validator's auto-funnel (catch-all) consumed all non-strict events so
+					// the bug was invisible. The TODO #10 `useFunnel` gate redirects some
+					// budget-boundary iterations to standalone, exposing the gap.
+					const data = await makeEvent(context, distinct_id, standaloneEarliest, u.pick(weightedEvents), user.anonymousIds, config.superProps || {}, config.groupKeys, false, false, standaloneFeatureCtx, standaloneIdentityCtx);
 					numEventsPreformed++;
 					newEvents = [data];
 				}
@@ -630,6 +682,14 @@ export async function userLoop(context) {
 				applyTouchpointCap(usersEvents, config, defaults, chance);
 			}
 
+			// v1.5.1: anonymous non-converters never call $identify in production,
+			// so Mixpanel never creates a profile for them. Stamp `_drop: true` so
+			// mixpanel-sender skips the /engage push. Stamped BEFORE the everything
+			// hook fires so hooks can rescue a profile by deleting the flag.
+			if (userIsBornInDataset && !userAuthed) {
+				profile._drop = true;
+			}
+
 			// Hook for processing all user events (hooks override everything)
 			if (config.hook) {
 				// `meta.isPreAuth(event)` predicate bound to this user's auth state.
@@ -691,8 +751,11 @@ export async function userLoop(context) {
 				console.warn(`⚠️  Dropped ${droppedFuture} future-dated event(s) for user ${distinct_id}`);
 			}
 
-			// Store all user data
-			await userProfilesData.hookPush(profile);
+			// Store all user data (skip profile push when a hook returned null
+			// for type='user' — see dropUserProfile above).
+			if (!dropUserProfile) {
+				await userProfilesData.hookPush(profile);
+			}
 
 			if (Object.keys(userSCD).length) {
 				for (const [key, changesArray] of Object.entries(userSCD)) {
@@ -714,10 +777,24 @@ export async function userLoop(context) {
 		});
 
 		userPromises.push(userPromise);
+
+		// v1.5.1: V8's Promise.all has a hard ceiling (~65K elements). Sharded
+		// kodiak runs pass numUsers in the millions per chunk, which blows that
+		// limit. Drain the in-flight buffer in batches of 50K to keep Promise.all
+		// well under the cap. With p-limit honoring `concurrency`, this is purely
+		// a backpressure boundary — no behavior change for existing dungeons that
+		// have numUsers under 50K.
+		if (userPromises.length >= 50_000) {
+			await Promise.all(userPromises);
+			userPromises.length = 0;
+		}
 	}
 
-	// Wait for all users to complete
-	await Promise.all(userPromises);
+	// Drain whatever's left in the final partial batch.
+	if (userPromises.length > 0) {
+		await Promise.all(userPromises);
+		userPromises.length = 0;
+	}
 
 	// Feature 4: Generate bot users (after regular users)
 	if (dataQuality && dataQuality.botUsers > 0) {
@@ -774,11 +851,15 @@ export function matchConditions(profile, conditions) {
  * @param {number} args.userActiveDays - Capacity (max possible distinct days)
  * @param {number} args.numEvents - Total events to schedule for this user
  * @param {number[]} [args.dowWeights] - 7-element soup DOW weights (Sun..Sat)
+ * @param {import('../utils/retention-curve.js').RetentionCurveConfig} [args.retentionCurve]
+ *   v1.5.1 — when set, weights candidate days by curve(day_offset_from_birth)
+ *   instead of DOW weights AND derives the effective avgActiveDaysPerUser from
+ *   the curve's sum across the window.
  * @param {Object} args.chance - Seeded chance instance
  * @returns {{ plan: number[], pickedDayBuckets: Set<number> } | null} Day plan
  *   + bucket set, or null if not applicable (no candidate days, no events, etc.).
  */
-function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, chance }) {
+function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDaysPerUser, userActiveDays, numEvents, dowWeights, retentionCurve, chance }) {
 	if (!Number.isFinite(numEvents) || numEvents <= 0) return null;
 
 	// Candidate day buckets: UTC days intersecting [max(adjustedCreated, FIXED_BEGIN), FIXED_NOW].
@@ -800,16 +881,36 @@ function buildActiveDayPlan({ adjustedCreated, fixedBegin, fixedNow, avgActiveDa
 	}
 	if (!candidateDays.length) return null;
 
-	// Weight by soup DOW (Sun=0..Sat=6). Default uniform if no weights configured.
-	const weights = candidateDays.map(daySec => {
-		if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
-		const dow = new Date(daySec * 1000).getUTCDay();
-		const w = Number(dowWeights[dow]);
-		return Number.isFinite(w) && w > 0 ? w : 0.0001;
-	});
+	// v1.5.1 (TODO #4): when `retentionCurve` is set, weight candidate days by
+	// curve(day_offset_from_birth) — biases active-day selection toward the
+	// requested retention shape. Replaces DOW weighting entirely (curve wins).
+	// Otherwise: legacy DOW weights.
+	let weights;
+	if (retentionCurve) {
+		const curveFn = buildCurveWeightFn(retentionCurve);
+		const birthDay = Math.floor(userStartUnix / dayMs);
+		weights = candidateDays.map(daySec => {
+			const dayOffset = (daySec / dayMs) - birthDay;
+			const w = curveFn(dayOffset);
+			return Number.isFinite(w) && w > 0 ? w : 0.0001;
+		});
+	} else {
+		weights = candidateDays.map(daySec => {
+			if (!Array.isArray(dowWeights) || dowWeights.length !== 7) return 1;
+			const dow = new Date(daySec * 1000).getUTCDay();
+			const w = Number(dowWeights[dow]);
+			return Number.isFinite(w) && w > 0 ? w : 0.0001;
+		});
+	}
 
 	// Draw target active-day count: normal(mean, sd=mean/3), clamped.
-	const meanActive = Math.max(1, Math.min(avgActiveDaysPerUser, candidateDays.length));
+	// v1.5.1: when `retentionCurve` is set, the mean is derived from the curve's
+	// sum across the user's window (resolution (b): curve wins, explicit
+	// avgActiveDaysPerUser ignored).
+	const effectiveMean = retentionCurve
+		? Math.max(1, expectedActiveDays(retentionCurve, candidateDays.length))
+		: avgActiveDaysPerUser;
+	const meanActive = Math.max(1, Math.min(effectiveMean, candidateDays.length));
 	const sd = Math.max(0.5, meanActive / 3);
 	let targetActiveDays = Math.round(chance.normal({ mean: meanActive, dev: sd }));
 	targetActiveDays = Math.max(1, Math.min(targetActiveDays, candidateDays.length));