@ak--47/dungeon-master 1.0.0

package/lib/orchestrators/mixpanel-sender.js

@@ -0,0 +1,266 @@
+/**
+ * Mixpanel Sender Orchestrator module
+ * Handles sending all data types to Mixpanel
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import { comma, ls, rm } from "ak-tools";
+import * as u from "../utils/utils.js";
+import mp from "mixpanel-import";
+
+/**
+ * Sends the data to Mixpanel
+ * @param {Context} context - Context object containing config, storage, etc.
+ * @returns {Promise<Object>} Import results for all data types
+ */
+export async function sendToMixpanel(context) {
+	const { config, storage } = context;
+	const {
+		adSpendData,
+		eventData,
+		groupProfilesData,
+		scdTableData,
+		userProfilesData,
+		groupEventData
+	} = storage;
+
+	const {
+		token,
+		region,
+		writeToDisk = true,
+		format,
+		serviceAccount,
+		projectId,
+		serviceSecret
+	} = config;
+
+	const importResults = { events: {}, users: {}, groups: [] };
+	const isBATCH_MODE = context.isBatchMode();
+	_verbose = config.verbose !== false;
+
+	/** @type {import('mixpanel-import').Creds} */
+	const creds = { token };
+	const mpImportFormat = format === "json" ? "jsonl" : "csv";
+
+	/** @type {import('mixpanel-import').Options} */
+	const commonOpts = {
+		region,
+		fixData: true,
+		verbose: false,
+		forceStream: true,
+		strict: true,
+		epochEnd: dayjs().unix(),
+		dryRun: false,
+		abridged: false,
+		fixJson: false,
+		showProgress: !!config.verbose,
+		streamFormat: mpImportFormat,
+		workers: 35
+	};
+
+	log(`\n${'─'.repeat(50)}`);
+	log(`  Importing data to Mixpanel (${region})`);
+	log(`${'─'.repeat(50)}\n`);
+
+	// Import events
+	if (eventData?.length > 0 || isBATCH_MODE) {
+		log(`  Events`);
+		let eventDataToImport = u.deepClone(eventData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && eventData && eventData.length === 0);
+		if (shouldReadFromFiles && eventData?.getWriteDir) {
+			const writeDir = eventData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			eventDataToImport = files.filter(f => f.includes('-EVENTS'));
+		}
+		const imported = await mp(creds, eventDataToImport, {
+			recordType: "event",
+			...commonOpts,
+		});
+		log(`  -> ${comma(imported.success)} events sent\n`);
+		importResults.events = imported;
+	}
+
+	// Import user profiles
+	if (userProfilesData?.length > 0 || isBATCH_MODE) {
+		log(`  User Profiles`);
+		let userProfilesToImport = u.deepClone(userProfilesData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && userProfilesData && userProfilesData.length === 0);
+		if (shouldReadFromFiles && userProfilesData?.getWriteDir) {
+			const writeDir = userProfilesData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			userProfilesToImport = files.filter(f => f.includes('-USERS'));
+		}
+		const imported = await mp(creds, userProfilesToImport, {
+			recordType: "user",
+			...commonOpts,
+		});
+		log(`  -> ${comma(imported.success)} user profiles sent\n`);
+		importResults.users = imported;
+	}
+
+	// Import ad spend data
+	if (adSpendData?.length > 0 || isBATCH_MODE) {
+		log(`  Ad Spend`);
+		let adSpendDataToImport = u.deepClone(adSpendData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && adSpendData && adSpendData.length === 0);
+		if (shouldReadFromFiles && adSpendData?.getWriteDir) {
+			const writeDir = adSpendData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			adSpendDataToImport = files.filter(f => f.includes('-ADSPEND'));
+		}
+		const imported = await mp(creds, adSpendDataToImport, {
+			recordType: "event",
+			...commonOpts,
+		});
+		log(`  -> ${comma(imported.success)} ad spend events sent\n`);
+		importResults.adSpend = imported;
+	}
+
+	// Import group profiles
+	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
+		for (const groupEntity of groupProfilesData) {
+			if (!groupEntity || groupEntity.length === 0) continue;
+			const groupKey = groupEntity?.groupKey;
+			log(`  Group Profiles (${groupKey})`);
+			let groupProfilesToImport = u.deepClone(groupEntity);
+			const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEntity.length === 0);
+			if (shouldReadFromFiles && groupEntity?.getWriteDir) {
+				const writeDir = groupEntity.getWriteDir();
+				const files = await ls(writeDir);
+				// @ts-ignore
+				groupProfilesToImport = files.filter(f => f.includes(`-${groupKey}-GROUPS`));
+			}
+			const imported = await mp({ token, groupKey }, groupProfilesToImport, {
+				recordType: "group",
+				...commonOpts,
+				groupKey,
+			});
+			log(`  -> ${comma(imported.success)} ${groupKey} profiles sent\n`);
+			importResults.groups.push(imported);
+		}
+	}
+
+	// Import group events
+	if (groupEventData?.length > 0) {
+		log(`  Group Events`);
+		let groupEventDataToImport = u.deepClone(groupEventData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEventData.length === 0);
+		if (shouldReadFromFiles && groupEventData?.getWriteDir) {
+			const writeDir = groupEventData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS'));
+		}
+		const imported = await mp(creds, groupEventDataToImport, {
+			recordType: "event",
+			...commonOpts,
+			strict: false
+		});
+		log(`  -> ${comma(imported.success)} group events sent\n`);
+		importResults.groupEvents = imported;
+	}
+
+	// Import SCD data (requires service account)
+	if (serviceAccount && projectId && serviceSecret) {
+		if (scdTableData && Array.isArray(scdTableData) && scdTableData.length > 0) {
+			for (const scdEntity of scdTableData) {
+				const scdKey = scdEntity?.scdKey;
+				const entityType = scdEntity?.entityType || 'user';
+				log(`  SCD: ${scdKey}`);
+				let scdDataToImport = u.deepClone(scdEntity);
+				const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && scdEntity && scdEntity.length === 0);
+				if (shouldReadFromFiles && scdEntity?.getWriteDir) {
+					const writeDir = scdEntity.getWriteDir();
+					const files = await ls(writeDir);
+					// @ts-ignore
+					scdDataToImport = files.filter(f => f.includes(`-${scdKey}-SCD`))?.pop();
+				}
+
+				/** @type {"string" | "number" | "boolean"} */
+				let scdType = 'string';
+				const scdExamplesValues = context.config.scdProps[Object.keys(context.config.scdProps).find(k => k === scdKey)].values;
+				if (scdExamplesValues) {
+					if (typeof scdExamplesValues[0] === 'number') {
+						scdType = 'number';
+					} else if (typeof scdExamplesValues[0] === 'boolean') {
+						scdType = 'boolean';
+					}
+				}
+
+				/** @type {import('mixpanel-import').Options} */
+				const options = {
+					recordType: "scd",
+					scdKey,
+					scdType,
+					scdLabel: `${scdKey}`,
+					fixData: true,
+					...commonOpts,
+				};
+
+				if (entityType !== "user") {
+					options.groupKey = entityType;
+				}
+
+				try {
+					const imported = await mp(
+						{
+							token,
+							acct: serviceAccount,
+							pass: serviceSecret,
+							project: projectId
+						},
+						scdDataToImport,
+						options
+					);
+					log(`  -> ${comma(imported.success)} ${scdKey} SCD entries sent\n`);
+					importResults[`${scdKey}_scd`] = imported;
+				} catch (err) {
+					log(`  !! failed: ${scdKey} SCD — ${err.message}\n`);
+					importResults[`${scdKey}_scd`] = { success: 0, failed: 0, error: err.message };
+				}
+			}
+		}
+	}
+
+	log(`${'─'.repeat(50)}\n`);
+
+	// Clean up batch files if needed
+	if (!writeToDisk && isBATCH_MODE) {
+		const writeDir = eventData?.getWriteDir?.() || userProfilesData?.getWriteDir?.();
+		if (writeDir) {
+			const configName = context.config.name;
+			const listDir = await ls(writeDir);
+			// @ts-ignore
+			const files = listDir.filter(f => {
+				if (configName && !f.includes(configName)) return false;
+				return f.includes('-EVENTS') ||
+					f.includes('-USERS') ||
+					f.includes('-ADSPEND') ||
+					f.includes('-GROUPS') ||
+					f.includes('-GROUP-EVENTS') ||
+					f.includes('-SCD') ||
+					f.includes('-MIRROR') ||
+					f.includes('-LOOKUP');
+			});
+			for (const file of files) {
+				await rm(file);
+			}
+		}
+	}
+
+	return importResults;
+}
+
+/**
+ * Logging function that respects verbose config
+ * @param {string} message - Message to log
+ */
+let _verbose = true;
+function log(message) {
+	if (_verbose) console.log(message);
+}

package/lib/orchestrators/user-loop.js

@@ -0,0 +1,335 @@
+/**
+ * User Loop Orchestrator module
+ * Manages user generation and event creation workflow
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import pLimit from 'p-limit';
+import os from 'os';
+import * as u from "../utils/utils.js";
+import * as t from 'ak-tools';
+import { makeEvent } from "../generators/events.js";
+import { makeFunnel } from "../generators/funnels.js";
+import { makeUserProfile } from "../generators/profiles.js";
+import { makeSCD } from "../generators/scd.js";
+
+/**
+ * Main user generation loop that creates users, their profiles, events, and SCDs
+ * @param {Context} context - Context object containing config, defaults, storage, etc.
+ * @returns {Promise<void>}
+ */
+export async function userLoop(context) {
+	const { config, storage, defaults } = context;
+	const chance = u.getChance();
+	const concurrency = config?.concurrency ?? 1;
+	const USER_CONN = pLimit(concurrency);
+
+	const {
+		verbose,
+		numUsers,
+		numEvents,
+		isAnonymous,
+		hasAvatar,
+		hasAnonIds,
+		hasSessionIds,
+		hasLocation,
+		funnels,
+		userProps,
+		scdProps,
+		numDays,
+		percentUsersBornInDataset = 15,
+		strictEventCount = false,
+		bornRecentBias = 0.3, // 0 = uniform distribution, 1 = heavily biased toward recent births
+	} = config;
+
+	const { eventData, userProfilesData, scdTableData } = storage;
+	const avgEvPerUser = numEvents / numUsers;
+	const startTime = Date.now();
+
+	// Create batches for parallel processing
+	const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
+	const userPromises = [];
+
+	// Track if we've already logged the strict event count message
+	let hasLoggedStrictCountReached = false;
+
+	// Handle graceful shutdown on SIGINT (Ctrl+C)
+	let cancelled = false;
+	const onSigint = () => {
+		cancelled = true;
+		USER_CONN.clearQueue();
+		if (verbose) console.log(`\n\nStopping generation (Ctrl+C)...\n`);
+	};
+	process.on('SIGINT', onSigint);
+
+	for (let i = 0; i < numUsers; i++) {
+		const userPromise = USER_CONN(async () => {
+			// Bail out if cancelled
+			if (cancelled) return;
+
+			// Bail out early if strictEventCount is enabled and we've hit numEvents
+			if (strictEventCount && context.getEventCount() >= numEvents) {
+				if (verbose && !hasLoggedStrictCountReached) {
+					console.log(`\n\u2713 Reached target of ${numEvents.toLocaleString()} events with strict event count enabled. Stopping user generation.`);
+					hasLoggedStrictCountReached = true;
+				}
+				return;
+			}
+
+			context.incrementUserCount();
+			const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
+			const memUsed = u.bytesHuman(process.memoryUsage().heapUsed);
+			const duration = u.formatDuration(Date.now() - startTime);
+
+			if (verbose) {
+				u.progress([
+					["users", context.getUserCount()],
+					["events", context.getEventCount()],
+					["eps", eps],
+					["mem", memUsed],
+					["time", duration]
+				]);
+			}
+
+			const userId = chance.guid();
+			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
+			const { distinct_id, created } = user;
+			const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
+			let numEventsPreformed = 0;
+
+			if (!userIsBornInDataset) delete user.created;
+
+			// Calculate time adjustments
+			const daysShift = context.getDaysShift();
+
+			// Apply recency bias to birth dates for users born in dataset
+			// bornRecentBias: 0 = uniform distribution, 1 = heavily biased toward recent
+			let adjustedCreated;
+			if (userIsBornInDataset) {
+				let biasedCreated = dayjs(created).subtract(daysShift, 'd');
+
+				if (bornRecentBias !== 0) {
+					// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
+					const datasetStart = dayjs.unix(global.FIXED_BEGIN);
+					const datasetEnd = dayjs.unix(context.FIXED_NOW);
+					const totalDuration = datasetEnd.diff(datasetStart);
+					// Clamp userPosition to [0, 1] to handle edge cases from rounding in time calculations
+					const userPosition = Math.max(0, Math.min(1, biasedCreated.diff(datasetStart) / totalDuration));
+
+					let biasedPosition;
+					if (bornRecentBias > 0) {
+						// Positive bias: exponent < 1 shifts distribution toward 1 (recent)
+						const exponent = 1 - (bornRecentBias * 0.7); // 0.3 bias -> 0.79 exponent (gentle nudge)
+						biasedPosition = Math.pow(userPosition, exponent);
+					} else {
+						// Negative bias: mirror the power function to shift toward 0 (early)
+						// -0.3 bias -> 0.79 exponent applied to (1 - position), then mirrored back
+						const exponent = 1 - (Math.abs(bornRecentBias) * 0.7);
+						biasedPosition = 1 - Math.pow(1 - userPosition, exponent);
+					}
+
+					// Convert back to timestamp
+					biasedCreated = datasetStart.add(biasedPosition * totalDuration, 'millisecond');
+				}
+
+				adjustedCreated = biasedCreated;
+				// Update user.created to match biased timestamp for profile consistency
+				user.created = adjustedCreated.toISOString();
+			} else {
+				adjustedCreated = dayjs.unix(global.FIXED_BEGIN);
+			}
+
+			if (hasLocation) {
+				const location = u.pickRandom(u.choose(defaults.locationsUsers));
+				for (const key in location) {
+					user[key] = location[key];
+				}
+			}
+
+			// Profile creation
+			const profile = await makeUserProfile(context, userProps, user);
+
+			// Call user hook after profile creation
+			if (config.hook) {
+				await config.hook(profile, "user", {
+					user,
+					config,
+					userIsBornInDataset
+				});
+			}
+
+			// SCD creation
+			// @ts-ignore
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
+			const scdTableKeys = Object.keys(scdUserTables);
+
+			const userSCD = {};
+			for (const [index, key] of scdTableKeys.entries()) {
+				const { max = 10 } = scdProps[key];
+				const mutations = chance.integer({ min: 1, max });
+				let changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
+				userSCD[key] = changes;
+
+				const hookResult = await config.hook(changes, "scd-pre", {
+					profile,
+					type: 'user',
+					scd: { [key]: scdProps[key] },
+					config,
+					allSCDs: userSCD
+				});
+				if (Array.isArray(hookResult)) {
+					changes = hookResult;
+					userSCD[key] = changes;
+				}
+			}
+
+			let numEventsThisUserWillPreform = Math.floor(chance.normal({
+				mean: avgEvPerUser,
+				dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
+			}) * 0.714159265359);
+
+			// Power users and low-activity users logic
+			chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
+			chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
+			numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
+
+			let userFirstEventTime;
+
+			const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+			const usageFunnels = funnels.filter((f) => !f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+
+			const secondsInDay = 86400;
+			const noise = () => chance.integer({ min: 0, max: secondsInDay });
+			let usersEvents = [];
+			let userConverted = true;
+
+			// Pre-compute weighted events array for standalone event selection
+			const weightedEvents = config.events.reduce((acc, event) => {
+				const w = Math.max(1, Math.min(Math.floor(event.weight) || 1, 10));
+				for (let i = 0; i < w; i++) acc.push(event);
+				return acc;
+			}, []);
+
+			// Build churn event lookup: { eventName: returnLikelihood }
+			const churnEvents = new Map();
+			for (const ev of config.events) {
+				if (ev.isChurnEvent) {
+					churnEvents.set(ev.event, ev.returnLikelihood ?? 0);
+				}
+			}
+
+			// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
+			if (firstFunnels.length && userIsBornInDataset) {
+				const firstFunnel = chance.pickone(firstFunnels, user);
+				const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+				const [data, converted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
+				userConverted = converted;
+
+				const timeShift = context.getTimeShift();
+				userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
+				numEventsPreformed += data.length;
+				usersEvents = usersEvents.concat(data);
+			} else {
+				userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+			}
+
+			// ALL SUBSEQUENT EVENTS (funnels for converted users, standalone for all)
+			let userChurned = false;
+			while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
+				let newEvents;
+				if (usageFunnels.length && userConverted) {
+					const currentFunnel = chance.pickone(usageFunnels);
+					const [data, converted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
+					numEventsPreformed += data.length;
+					newEvents = data;
+				} else {
+					const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(weightedEvents), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
+					numEventsPreformed++;
+					newEvents = [data];
+				}
+				usersEvents = usersEvents.concat(newEvents);
+
+				// Check for churn events — if user churned, they may stop generating
+				if (churnEvents.size > 0) {
+					const eventsToCheck = Array.isArray(newEvents[0]) ? newEvents.flat() : newEvents;
+					for (const ev of eventsToCheck) {
+						if (ev.event && churnEvents.has(ev.event)) {
+							const returnLikelihood = churnEvents.get(ev.event);
+							const userReturns = returnLikelihood > 0 && chance.bool({ likelihood: returnLikelihood * 100 });
+							if (!userReturns) {
+								userChurned = true;
+								break;
+							}
+						}
+					}
+					if (userChurned) break;
+				}
+			}
+
+			// Remove events flagged as future timestamps (before dungeon hooks see them)
+			usersEvents = usersEvents.filter(e => !e._drop);
+
+			// Hook for processing all user events
+			if (config.hook) {
+				const newEvents = await config.hook(usersEvents, "everything", {
+					profile,
+					scd: userSCD,
+					config,
+					userIsBornInDataset
+				});
+				if (Array.isArray(newEvents)) usersEvents = newEvents;
+			}
+
+			// Store all user data
+			await userProfilesData.hookPush(profile);
+
+			if (Object.keys(userSCD).length) {
+				for (const [key, changesArray] of Object.entries(userSCD)) {
+					for (const changes of changesArray) {
+						try {
+							const target = scdTableData.filter(arr => arr.scdKey === key).pop();
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+						catch (e) {
+							// This is probably a test
+							const target = scdTableData[0];
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+					}
+				}
+			}
+
+			await eventData.hookPush(usersEvents, { profile });
+		});
+
+		userPromises.push(userPromise);
+	}
+
+	// Wait for all users to complete
+	await Promise.all(userPromises);
+
+	// Clean up SIGINT handler
+	process.removeListener('SIGINT', onSigint);
+}
+
+
+export function weighFunnels(acc, funnel) {
+	const weight = funnel?.weight || 1;
+	for (let i = 0; i < weight; i++) {
+		acc.push(funnel);
+	}
+	return acc;
+}
+
+export function matchConditions(profile, conditions) {
+	for (const [key, value] of Object.entries(conditions)) {
+		if (profile[key] !== value) return false;
+	}
+	return true;
+}