make-mp-data 2.0.21 → 2.0.23

package/lib/core/storage.js

@@ -32,13 +32,24 @@ export async function createHookArray(arr = [], opts) {
 	} = opts || {};
 
 	const FILE_CONN = pLimit(concurrency);
-	const { config = {}, runtime = {} } = context;
+	const { 
+		config = {}, 
+		runtime = { 
+			operations: 0, 
+			eventCount: 0, 
+			userCount: 0, 
+			isBatchMode: false, 
+			verbose: false, 
+			isCLI: false 
+		} 
+	} = context;
 	const BATCH_SIZE = config.batchSize || 1_000_000;
 	const NODE_ENV = process.env.NODE_ENV || "unknown";
 
 	let batch = 0;
 	let writeDir;
 	let isBatchMode = runtime.isBatchMode || false;
+	let isWriting = false; // Prevent concurrent writes
 
 	// Determine write directory
 	const dataFolder = path.resolve("./data");
@@ -54,13 +65,15 @@ export async function createHookArray(arr = [], opts) {
 	}
 
 	function getWritePath() {
+		const gzipSuffix = (config.gzip && !writeDir?.startsWith('gs://')) ? '.gz' : '';
+		
 		if (isBatchMode) {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
-			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`);
 		}
 		else {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
-			return path.join(writeDir, `${filepath}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}.${format}${gzipSuffix}`);
 		}
 	}
 
@@ -109,15 +122,26 @@ export async function createHookArray(arr = [], opts) {
 			}
 		}
 
-		if (arr.length > BATCH_SIZE) {
+		// Check batch size and handle writes synchronously to prevent race conditions
+		if (arr.length > BATCH_SIZE && !isWriting) {
+			isWriting = true; // Lock to prevent concurrent writes
 			isBatchMode = true;
 			runtime.isBatchMode = true; // Update runtime state
 			batch++;
 			const writePath = getWritePath();
-			const writeResult = await FILE_CONN(() => writeToDisk(arr, { writePath }));
-			// Ensure array is cleared after successful write
-			arr.length = 0;
-			return writeResult;
+			
+			try {
+				// Create a copy of the data to write
+				const dataToWrite = [...arr];
+				// Clear the array immediately to prevent race conditions
+				arr.length = 0;
+				
+				// Write to disk (this is now synchronous from the perspective of batch management)
+				const writeResult = await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
+				return writeResult;
+			} finally {
+				isWriting = false; // Release the lock
+			}
 		} else {
 			return Promise.resolve(false);
 		}
@@ -131,12 +155,19 @@ export async function createHookArray(arr = [], opts) {
 			console.log(`\n\twriting ${writePath}\n`);
 		}
 
+		const streamOptions = {
+			gzip: config.gzip || false
+		};
+
 		switch (format) {
 			case "csv":
-				writeResult = await u.streamCSV(writePath, data);
+				writeResult = await u.streamCSV(writePath, data, streamOptions);
 				break;
 			case "json":
-				writeResult = await u.streamJSON(writePath, data);
+				writeResult = await u.streamJSON(writePath, data, streamOptions);
+				break;
+			case "parquet":
+				writeResult = await u.streamParquet(writePath, data, streamOptions);
 				break;
 			default:
 				throw new Error(`format ${format} is not supported`);
@@ -148,10 +179,21 @@ export async function createHookArray(arr = [], opts) {
 
 	async function flush() {
 		if (arr.length > 0) {
-			batch++;
-			const writePath = getWritePath();
-			await FILE_CONN(() => writeToDisk(arr, { writePath }));
-			if (isBatchMode) arr.length = 0; // free up memory for batch mode
+			// Wait for any ongoing writes to complete
+			while (isWriting) {
+				await new Promise(resolve => setTimeout(resolve, 10));
+			}
+			
+			isWriting = true;
+			try {
+				batch++;
+				const writePath = getWritePath();
+				const dataToWrite = [...arr];
+				arr.length = 0; // Clear array after copying data
+				await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
+			} finally {
+				isWriting = false;
+			}
 		}
 	}
 
@@ -186,12 +228,15 @@ export class StorageManager {
 	async initializeContainers() {
 		const { config } = this.context;
 
+		// Validate configuration for potential data loss scenarios
+		this.validateConfiguration(config);
+
 		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
 				type: "event",
-				filepath: `${config.simulationName || 'events'}-EVENTS`,
+				filepath: `${config.name}-EVENTS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -200,7 +245,7 @@ export class StorageManager {
 			userProfilesData: await createHookArray([], {
 				hook: config.hook,
 				type: "user",
-				filepath: `${config.simulationName || 'users'}-USERS`,
+				filepath: `${config.name}-USERS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -209,7 +254,7 @@ export class StorageManager {
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
 				type: "ad-spend",
-				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
+				filepath: `${config.name}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -222,7 +267,7 @@ export class StorageManager {
 			mirrorEventData: await createHookArray([], {
 				hook: config.hook,
 				type: "mirror",
-				filepath: `${config.simulationName || 'mirror'}-MIRROR`,
+				filepath: `${config.name}-MIRROR`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -235,7 +280,7 @@ export class StorageManager {
 				const scdArray = await createHookArray([], {
 					hook: config.hook,
 					type: "scd",
-					filepath: `${config.simulationName || 'scd'}-${scdKey}-SCD`,
+					filepath: `${config.name}-${scdKey}-SCD`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -251,7 +296,7 @@ export class StorageManager {
 				const groupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "group",
-					filepath: `${config.simulationName || 'groups'}-${groupKey}-GROUPS`,
+					filepath: `${config.name}-${groupKey}-GROUPS`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -267,7 +312,7 @@ export class StorageManager {
 				const lookupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "lookup",
-					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
+					filepath: `${config.name}-${lookupConfig.key}-LOOKUP`,
 					format: "csv", // Always force CSV for lookup tables
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -279,4 +324,24 @@ export class StorageManager {
 
 		return storage;
 	}
+
+	/**
+	 * Validates configuration to prevent data loss scenarios
+	 * @param {Object} config - Configuration object
+	 */
+	validateConfiguration(config) {
+		// Check for potential data loss scenario: writeToDisk=false with low batchSize
+		if (config.writeToDisk === false) {
+			const batchSize = config.batchSize || 1_000_000;
+			const numEvents = config.numEvents || 0;
+			
+			if (batchSize < numEvents) {
+				throw new Error(
+					`Configuration error: writeToDisk is explicitly set to false but batchSize (${batchSize}) is lower than numEvents (${numEvents}). ` +
+					`This would result in data loss as batched data would be discarded. ` +
+					`Either set writeToDisk to true, increase batchSize to be >= numEvents, or provide a Mixpanel token to send data directly.`
+				);
+			}
+		}
+	}
 }

package/lib/generators/events.js

@@ -72,7 +72,6 @@ export async function makeEvent(
     };
 
     let defaultProps = {};
-    let devicePool = [];
 
     // Add default properties based on configuration
     if (hasLocation) {
@@ -82,32 +81,30 @@ export async function makeEvent(
     if (hasBrowser) {
         defaultProps.browser = u.choose(defaults.browsers());
     }
-    
-    // Build device pool based on enabled device types
-    if (hasAndroidDevices) devicePool.push(defaults.androidDevices());
-    if (hasIOSDevices) devicePool.push(defaults.iOSDevices());
-    if (hasDesktopDevices) devicePool.push(defaults.desktopDevices());
 
     // Add campaigns with attribution likelihood
     if (hasCampaigns && chance.bool({ likelihood: 25 })) {
         defaultProps.campaigns = u.pickRandom(defaults.campaigns());
     }
     
-    // Select device from pool
-    const devices = devicePool.flat();
-    if (devices.length) {
-        defaultProps.device = u.pickRandom(devices);
+    // PERFORMANCE: Use pre-computed device pool instead of rebuilding every time
+    if (defaults.allDevices.length) {
+        defaultProps.device = u.pickRandom(defaults.allDevices);
     }
 
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            // Apply time shift to move to present day using precomputed value
-            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Direct numeric calculation instead of dayjs object creation
+            const shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         } else {
-            // Get time from TimeSoup and apply precomputed time shift
+            // Get time from TimeSoup (returns ISO string) and apply precomputed time shift
             const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
-            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Parse ISO directly to milliseconds, add shift, convert back to ISO with one dayjs call
+            const soupTimestamp = new Date(soupTime).getTime() / 1000; // Convert to unix seconds
+            const shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         }
     }
 
@@ -133,7 +130,9 @@ export async function makeEvent(
     // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
     if (chosenEvent.properties) {
-        for (const key in chosenEvent.properties) {
+        const eventKeys = Object.keys(chosenEvent.properties);
+        for (let i = 0; i < eventKeys.length; i++) {
+            const key = eventKeys[i];
             try {
                 eventTemplate[key] = u.choose(chosenEvent.properties[key]);
             } catch (e) {
@@ -145,7 +144,9 @@ export async function makeEvent(
     
     // Add super properties (override event properties if needed)
     if (superProps) {
-        for (const key in superProps) {
+        const superKeys = Object.keys(superProps);
+        for (let i = 0; i < superKeys.length; i++) {
+            const key = superKeys[i];
             try {
                 eventTemplate[key] = u.choose(superProps[key]);
             } catch (e) {

package/lib/generators/funnels.js

@@ -128,7 +128,13 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
     
     return sequence.map((eventName) => {
         const foundEvent = config.events?.find((e) => e.event === eventName);
-        const eventSpec = u.deepClone(foundEvent) || { event: eventName, properties: {} };
+        
+        // PERFORMANCE: Shallow copy instead of deepClone for better performance
+        // We only need to copy the top-level structure since we're rebuilding properties anyway
+        const eventSpec = foundEvent ? {
+            event: foundEvent.event,
+            properties: { ...foundEvent.properties }
+        } : { event: eventName, properties: {} };
         
         // Process event properties
         for (const key in eventSpec.properties) {
@@ -139,11 +145,7 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
             }
         }
         
-        // Clean up funnel-specific properties
-        delete eventSpec.isFirstEvent;
-        delete eventSpec.weight;
-        
-        // Merge funnel properties
+        // Merge funnel properties (no need to delete properties since we're creating a new object)
         eventSpec.properties = { ...eventSpec.properties, ...chosenFunnelProps };
         
         return eventSpec;

package/lib/orchestrators/mixpanel-sender.js

@@ -58,9 +58,10 @@ export async function sendToMixpanel(context) {
         epochEnd: dayjs().unix(),
         dryRun: false,
         abridged: false,
-        fixJson: true,
+        fixJson: false,
         showProgress: NODE_ENV === "dev" ? true : false,
-        streamFormat: mpImportFormat
+        streamFormat: mpImportFormat,
+		workers: 35
     };
 
     if (isCLI) commonOpts.showProgress = true;
@@ -134,6 +135,8 @@ export async function sendToMixpanel(context) {
             const imported = await mp({ token, groupKey }, groupProfilesToImport, {
                 recordType: "group",
                 ...commonOpts,
+				groupKey,
+				//dryRun: true
             });
             log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
             importResults.groups.push(imported);