make-mp-data 2.1.6 → 3.0.1

package/lib/orchestrators/mixpanel-sender.js

@@ -17,214 +17,257 @@ import mp from "mixpanel-import";
  * @returns {Promise<Object>} Import results for all data types
  */
 export async function sendToMixpanel(context) {
-    const { config, storage } = context;
-    const {
-        adSpendData,
-        eventData,
-        groupProfilesData,
-        lookupTableData,
-        mirrorEventData,
-        scdTableData,
-        userProfilesData,
-        groupEventData
-    } = storage;
-
-    const {
-        token,
-        region,
-        writeToDisk = true,
-        format,
-        serviceAccount,
-        projectId,
-        serviceSecret,
-        verbose = false
-    } = config;
-
-    const importResults = { events: {}, users: {}, groups: [] };
-    const isBATCH_MODE = context.isBatchMode();
-    const isCLI = context.isCLI();
-    const NODE_ENV = process.env.NODE_ENV || "unknown";
-
-    // Create verbose-aware log function
-    const log = (message) => {
-        if (verbose) console.log(message);
-    };
-
-    /** @type {import('mixpanel-import').Creds} */
-    const creds = { token };
-    const mpImportFormat = format === "json" ? "jsonl" : "csv";
-    
-    /** @type {import('mixpanel-import').Options} */
-    const commonOpts = {
-        region,
-        fixData: true,
-        verbose: false,
-        forceStream: true,
-        strict: true,
-        epochEnd: dayjs().unix(),
-        dryRun: false,
-        abridged: false,
-        fixJson: false,
-        showProgress: verbose,
-        streamFormat: mpImportFormat,
-        workers: 35
-    };
-
-    // Import events
-    if (eventData || isBATCH_MODE) {
-        log(`importing events to mixpanel...\n`);
-        let eventDataToImport = u.deepClone(eventData);
-        if (isBATCH_MODE) {
-            const writeDir = eventData.getWriteDir();
-            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-            // @ts-ignore
-            eventDataToImport = files.filter(f => f.includes('-EVENTS-'));
-        }
-        const imported = await mp(creds, eventDataToImport, {
-            recordType: "event",
-            ...commonOpts,
-        });
-        log(`\tsent ${comma(imported.success)} events\n`);
-        importResults.events = imported;
-    }
-
-    // Import user profiles
-    if (userProfilesData || isBATCH_MODE) {
-        log(`importing user profiles to mixpanel...\n`);
-        let userProfilesToImport = u.deepClone(userProfilesData);
-        if (isBATCH_MODE) {
-            const writeDir = userProfilesData.getWriteDir();
-            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-            // @ts-ignore
-            userProfilesToImport = files.filter(f => f.includes('-USERS-'));
-        }
-        const imported = await mp(creds, userProfilesToImport, {
-            recordType: "user",
-            ...commonOpts,
-        });
-        log(`\tsent ${comma(imported.success)} user profiles\n`);
-        importResults.users = imported;
-    }
-
-    // Import ad spend data  
-    if (adSpendData || isBATCH_MODE) {
-        log(`importing ad spend data to mixpanel...\n`);
-        let adSpendDataToImport = u.deepClone(adSpendData);
-        if (isBATCH_MODE) {
-            const writeDir = adSpendData.getWriteDir();
-            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-            // @ts-ignore
-            adSpendDataToImport = files.filter(f => f.includes('-AD-SPEND-'));
-        }
-        const imported = await mp(creds, adSpendDataToImport, {
-            recordType: "event",
-            ...commonOpts,
-        });
-        log(`\tsent ${comma(imported.success)} ad spend events\n`);
-        importResults.adSpend = imported;
-    }
-
-    // Import group profiles
-    if (groupProfilesData || isBATCH_MODE) {
-        for (const groupEntity of groupProfilesData) {
-            const groupKey = groupEntity?.groupKey;
-            log(`importing ${groupKey} profiles to mixpanel...\n`);
-            let groupProfilesToImport = u.deepClone(groupEntity);
-            if (isBATCH_MODE) {
-                const writeDir = groupEntity.getWriteDir();
-                const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-                // @ts-ignore
-                groupProfilesToImport = files.filter(f => f.includes(`-GROUPS-${groupKey}`));
-            }
-            const imported = await mp({ token, groupKey }, groupProfilesToImport, {
-                recordType: "group",
-                ...commonOpts,
+	const { config, storage } = context;
+	const {
+		adSpendData,
+		eventData,
+		groupProfilesData,
+		lookupTableData,
+		mirrorEventData,
+		scdTableData,
+		userProfilesData,
+		groupEventData
+	} = storage;
+
+	const {
+		token,
+		region,
+		writeToDisk = true,
+		format,
+		serviceAccount,
+		projectId,
+		serviceSecret
+	} = config;
+
+	const importResults = { events: {}, users: {}, groups: [] };
+	const isBATCH_MODE = context.isBatchMode();
+	const NODE_ENV = process.env.NODE_ENV || "unknown";
+
+	/** @type {import('mixpanel-import').Creds} */
+	const creds = { token };
+	const mpImportFormat = format === "json" ? "jsonl" : "csv";
+
+	const isDev = NODE_ENV !== 'production';
+
+	/** @type {import('mixpanel-import').Options} */
+	const commonOpts = {
+		region,
+		fixData: true,
+		verbose: isDev,
+		forceStream: true,
+		strict: true,
+		epochEnd: dayjs().unix(),
+		dryRun: false,
+		abridged: false,
+		fixJson: false,
+		showProgress: isDev,
+		streamFormat: mpImportFormat,
+		workers: 35
+	};
+
+	// Import events
+	if (eventData?.length > 0 || isBATCH_MODE) {
+		log(`importing events to mixpanel...\n`);
+		let eventDataToImport = u.deepClone(eventData);
+		// Check if we need to read from disk files instead of memory
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && eventData && eventData.length === 0);
+		if (shouldReadFromFiles && eventData?.getWriteDir) {
+			const writeDir = eventData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			eventDataToImport = files.filter(f => f.includes('-EVENTS'));
+		}
+		const imported = await mp(creds, eventDataToImport, {
+			recordType: "event",
+			...commonOpts,
+		});
+		log(`\tsent ${comma(imported.success)} events\n`);
+		importResults.events = imported;
+	}
+
+	// Import user profiles
+	if (userProfilesData?.length > 0 || isBATCH_MODE) {
+		log(`importing user profiles to mixpanel...\n`);
+		let userProfilesToImport = u.deepClone(userProfilesData);
+		// Check if we need to read from disk files instead of memory
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && userProfilesData && userProfilesData.length === 0);
+		if (shouldReadFromFiles && userProfilesData?.getWriteDir) {
+			const writeDir = userProfilesData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			userProfilesToImport = files.filter(f => f.includes('-USERS'));
+		}
+		const imported = await mp(creds, userProfilesToImport, {
+			recordType: "user",
+			...commonOpts,
+		});
+		log(`\tsent ${comma(imported.success)} user profiles\n`);
+		importResults.users = imported;
+	}
+
+	// Import ad spend data
+	if (adSpendData?.length > 0 || isBATCH_MODE) {
+		log(`importing ad spend data to mixpanel...\n`);
+		let adSpendDataToImport = u.deepClone(adSpendData);
+		// Check if we need to read from disk files instead of memory
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && adSpendData && adSpendData.length === 0);
+		if (shouldReadFromFiles && adSpendData?.getWriteDir) {
+			const writeDir = adSpendData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			adSpendDataToImport = files.filter(f => f.includes('-ADSPEND'));
+		}
+		const imported = await mp(creds, adSpendDataToImport, {
+			recordType: "event",
+			...commonOpts,
+		});
+		log(`\tsent ${comma(imported.success)} ad spend events\n`);
+		importResults.adSpend = imported;
+	}
+
+	// Import group profiles
+	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
+		for (const groupEntity of groupProfilesData) {
+			if (!groupEntity || groupEntity.length === 0) continue;
+			const groupKey = groupEntity?.groupKey;
+			log(`importing ${groupKey} profiles to mixpanel...\n`);
+			let groupProfilesToImport = u.deepClone(groupEntity);
+			// Check if we need to read from disk files instead of memory
+			const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEntity.length === 0);
+			if (shouldReadFromFiles && groupEntity?.getWriteDir) {
+				const writeDir = groupEntity.getWriteDir();
+				const files = await ls(writeDir);
+				// @ts-ignore
+				groupProfilesToImport = files.filter(f => f.includes(`-${groupKey}-GROUPS`));
+			}
+			const imported = await mp({ token, groupKey }, groupProfilesToImport, {
+				recordType: "group",
+				...commonOpts,
 				groupKey,
-				//dryRun: true
-            });
-            log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
-            importResults.groups.push(imported);
-        }
-    }
-
-    // Import group events
-    if (groupEventData || isBATCH_MODE) {
-        log(`importing group events to mixpanel...\n`);
-        let groupEventDataToImport = u.deepClone(groupEventData);
-        if (isBATCH_MODE) {
-            const writeDir = groupEventData.getWriteDir();
-            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-            // @ts-ignore
-            groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS-'));
-        }
-        const imported = await mp(creds, groupEventDataToImport, {
-            recordType: "event",
-            ...commonOpts,
-            strict: false
-        });
-        log(`\tsent ${comma(imported.success)} group events\n`);
-        importResults.groupEvents = imported;
-    }
-
-    // Import SCD data (requires service account)
-    if (serviceAccount && projectId && serviceSecret) {
-        if (scdTableData || isBATCH_MODE) {
-            log(`importing SCD data to mixpanel...\n`);
-            for (const scdEntity of scdTableData) {
-                const scdKey = scdEntity?.scdKey;
-                log(`importing ${scdKey} SCD data to mixpanel...\n`);
-                let scdDataToImport = u.deepClone(scdEntity);
-                if (isBATCH_MODE) {
-                    const writeDir = scdEntity.getWriteDir();
-                    const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
-                    // @ts-ignore
-                    scdDataToImport = files.filter(f => f.includes(`-SCD-${scdKey}`));
-                }
-
-                /** @type {import('mixpanel-import').Options} */
-                const options = {
-                    recordType: "scd",
-                    scdKey,
-                    scdType: scdEntity.dataType,
-                    scdLabel: `${scdKey}-scd`,
-                    ...commonOpts,
-                };
-
-                if (scdEntity.entityType !== "user") options.groupKey = scdEntity.entityType;
-                
-                const imported = await mp(
-                    {
-                        token,
-                        acct: serviceAccount,
-                        pass: serviceSecret,
-                        project: projectId
-                    },
-                    scdDataToImport,
-                    options
-                );
-                log(`\tsent ${comma(imported.success)} ${scdKey} SCD data\n`);
-                importResults[`${scdKey}_scd`] = imported;
-            }
-        }
-    }
-
-    // Clean up batch files if needed
-    if (!writeToDisk && isBATCH_MODE) {
-        const writeDir = eventData?.getWriteDir() || userProfilesData?.getWriteDir();
-        const listDir = await ls(writeDir.split(path.basename(writeDir)).join(""));
-        // @ts-ignore
-        const files = listDir.filter(f => 
-            f.includes('-EVENTS-') || 
-            f.includes('-USERS-') || 
-            f.includes('-AD-SPEND-') || 
-            f.includes('-GROUPS-') || 
-            f.includes('-GROUP-EVENTS-')
-        );
-        for (const file of files) {
-            await rm(file);
-        }
-    }
-    
-    return importResults;
+			});
+			log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
+			importResults.groups.push(imported);
+		}
+	}
+
+	// Import group events
+	if (groupEventData?.length > 0) {
+		log(`importing group events to mixpanel...\n`);
+		let groupEventDataToImport = u.deepClone(groupEventData);
+		// Check if we need to read from disk files instead of memory
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEventData.length === 0);
+		if (shouldReadFromFiles && groupEventData?.getWriteDir) {
+			const writeDir = groupEventData.getWriteDir();
+			const files = await ls(writeDir);
+			// @ts-ignore
+			groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS'));
+		}
+		const imported = await mp(creds, groupEventDataToImport, {
+			recordType: "event",
+			...commonOpts,
+			strict: false
+		});
+		log(`\tsent ${comma(imported.success)} group events\n`);
+		importResults.groupEvents = imported;
+	}
+
+	// Import SCD data (requires service account)
+	if (serviceAccount && projectId && serviceSecret) {
+		if (scdTableData && Array.isArray(scdTableData) && scdTableData.length > 0) {
+			log(`importing SCD data to mixpanel...\n`);
+			for (const scdEntity of scdTableData) {
+				const scdKey = scdEntity?.scdKey;
+				const entityType = scdEntity?.entityType || 'user';
+				log(`importing ${scdKey} SCD data to mixpanel...\n`);
+				let scdDataToImport = u.deepClone(scdEntity);
+				// Check if we need to read from disk files instead of memory
+				const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && scdEntity && scdEntity.length === 0);
+				if (shouldReadFromFiles && scdEntity?.getWriteDir) {
+					const writeDir = scdEntity.getWriteDir();
+					const files = await ls(writeDir);
+					// @ts-ignore
+					scdDataToImport = files.filter(f => f.includes(`-${scdKey}-SCD`))?.pop();
+
+				}
+
+				// Derive the data type from the actual SCD data
+				// todo: we can do better type inference here we don't need to visit the file
+				/** @type {"string" | "number" | "boolean"} */
+			let scdType = 'string'; // default to string
+				const scdExamplesValues = context.config.scdProps[Object.keys(context.config.scdProps).find(k => k === scdKey)].values;
+				if (scdExamplesValues) {
+					if (typeof scdExamplesValues[0] === 'number') {
+						scdType = 'number';
+					} else if (typeof scdExamplesValues[0] === 'boolean') {
+						scdType = 'boolean';
+					}
+				}
+
+
+
+				/** @type {import('mixpanel-import').Options} */
+				const options = {
+					recordType: "scd",
+					scdKey,
+					scdType,
+					scdLabel: `${scdKey}`,
+					fixData: true,
+					...commonOpts,
+				};
+
+				// For group SCDs, add the groupKey
+				if (entityType !== "user") {
+					options.groupKey = entityType;
+				}
+
+				// SCD data is sketch and it shouldn't fail the whole import
+				try {
+					const imported = await mp(
+						{
+							token,
+							acct: serviceAccount,
+							pass: serviceSecret,
+							project: projectId
+						},
+						scdDataToImport,
+						options
+					);
+					log(`\tsent ${comma(imported.success)} ${scdKey} SCD data\n`);
+					importResults[`${scdKey}_scd`] = imported;
+				} catch (err) {
+					log(`\tfailed to import ${scdKey} SCD data: ${err.message}\n`);
+					importResults[`${scdKey}_scd`] = { success: 0, failed: 0, error: err.message };
+				}
+			}
+		}
+	}
+
+	// Clean up batch files if needed
+	if (!writeToDisk && isBATCH_MODE) {
+		const writeDir = eventData?.getWriteDir?.() || userProfilesData?.getWriteDir?.();
+		if (writeDir) {
+			const listDir = await ls(writeDir);
+			// @ts-ignore
+			const files = listDir.filter(f =>
+				f.includes('-EVENTS') ||
+				f.includes('-USERS') ||
+				f.includes('-ADSPEND') ||
+				f.includes('-GROUPS') ||
+				f.includes('-GROUP-EVENTS')
+			);
+			for (const file of files) {
+				await rm(file);
+			}
+		}
+	}
+
+	return importResults;
 }
 
+/**
+ * Simple logging function
+ * @param {string} message - Message to log
+ */
+function log(message) {
+	console.log(message);
+}

package/lib/orchestrators/user-loop.js

@@ -39,7 +39,9 @@ export async function userLoop(context) {
 		userProps,
 		scdProps,
 		numDays,
-		percentUsersBornInDataset = 5,
+		percentUsersBornInDataset = 15,
+		strictEventCount = false,
+		bornRecentBias = 0.3, // 0 = uniform distribution, 1 = heavily biased toward recent births
 	} = config;
 
 	const { eventData, userProfilesData, scdTableData } = storage;
@@ -50,16 +52,32 @@ export async function userLoop(context) {
 	const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
 	const userPromises = [];
 
+	// Track if we've already logged the strict event count message
+	let hasLoggedStrictCountReached = false;
+
 	for (let i = 0; i < numUsers; i++) {
 		const userPromise = USER_CONN(async () => {
+			// Bail out early if strictEventCount is enabled and we've hit numEvents
+			if (strictEventCount && context.getEventCount() >= numEvents) {
+				if (verbose && !hasLoggedStrictCountReached) {
+					console.log(`\n\u2713 Reached target of ${numEvents.toLocaleString()} events with strict event count enabled. Stopping user generation.`);
+					hasLoggedStrictCountReached = true;
+				}
+				return;
+			}
+
 			context.incrementUserCount();
 			const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
+			const memUsed = u.bytesHuman(process.memoryUsage().heapUsed);
+			const duration = u.formatDuration(Date.now() - startTime);
 
 			if (verbose) {
 				u.progress([
 					["users", context.getUserCount()],
 					["events", context.getEventCount()],
-					["eps", eps]
+					["eps", eps],
+					["mem", memUsed],
+					["time", duration]
 				]);
 			}
 
@@ -73,9 +91,36 @@ export async function userLoop(context) {
 
 			// Calculate time adjustments
 			const daysShift = context.getDaysShift();
-			const adjustedCreated = userIsBornInDataset
-				? dayjs(created).subtract(daysShift, 'd')
-				: dayjs.unix(global.FIXED_BEGIN);
+
+			// Apply recency bias to birth dates for users born in dataset
+			// bornRecentBias: 0 = uniform distribution, 1 = heavily biased toward recent
+			let adjustedCreated;
+			if (userIsBornInDataset) {
+				let biasedCreated = dayjs(created).subtract(daysShift, 'd');
+
+				if (bornRecentBias > 0) {
+					// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
+					const datasetStart = dayjs.unix(global.FIXED_BEGIN);
+					const datasetEnd = dayjs.unix(context.FIXED_NOW);
+					const totalDuration = datasetEnd.diff(datasetStart);
+					// Clamp userPosition to [0, 1] to handle edge cases from rounding in time calculations
+					const userPosition = Math.max(0, Math.min(1, biasedCreated.diff(datasetStart) / totalDuration));
+
+					// Apply power function to bias toward recent (higher values)
+					// exponent < 1 shifts distribution toward 1 (recent)
+					const exponent = 1 - (bornRecentBias * 0.7); // 0.3 bias -> 0.79 exponent (gentle nudge)
+					const biasedPosition = Math.pow(userPosition, exponent);
+
+					// Convert back to timestamp
+					biasedCreated = datasetStart.add(biasedPosition * totalDuration, 'millisecond');
+				}
+
+				adjustedCreated = biasedCreated;
+				// Update user.created to match biased timestamp for profile consistency
+				user.created = adjustedCreated.toISOString();
+			} else {
+				adjustedCreated = dayjs.unix(global.FIXED_BEGIN);
+			}
 
 			if (hasLocation) {
 				const location = u.pickRandom(u.choose(defaults.locationsUsers));
@@ -89,10 +134,10 @@ export async function userLoop(context) {
 
 			// Call user hook after profile creation
 			if (config.hook) {
-				await config.hook(profile, "user", { 
-					user, 
+				await config.hook(profile, "user", {
+					user,
 					config,
-					userIsBornInDataset 
+					userIsBornInDataset
 				});
 			}
 
@@ -149,12 +194,9 @@ export async function userLoop(context) {
 				const timeShift = context.getTimeShift();
 				userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
 				numEventsPreformed += data.length;
-				usersEvents.push(...data);
+				usersEvents = usersEvents.concat(data);
 
 				if (!userConverted) {
-					// if (verbose) {
-					//     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-					// }
 					return;
 				}
 			} else {
@@ -167,11 +209,11 @@ export async function userLoop(context) {
 					const currentFunnel = chance.pickone(usageFunnels);
 					const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
 					numEventsPreformed += data.length;
-					usersEvents.push(...data);
+					usersEvents = usersEvents.concat(data);
 				} else {
 					const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
 					numEventsPreformed++;
-					usersEvents.push(data);
+					usersEvents = usersEvents.concat(data);
 				}
 			}
 
@@ -206,10 +248,6 @@ export async function userLoop(context) {
 			}
 
 			await eventData.hookPush(usersEvents, { profile });
-
-			if (verbose) {
-				// u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-			}
 		});
 
 		userPromises.push(userPromise);
@@ -233,4 +271,4 @@ export function matchConditions(profile, conditions) {
 		if (profile[key] !== value) return false;
 	}
 	return true;
-}
+}