make-mp-data 2.0.19 → 2.0.22

package/lib/core/storage.js

@@ -3,7 +3,10 @@
  * Extracted from index.js to eliminate global dependencies
  */
 
-/** @typedef {import('../../types').Context} Context */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').HookedArray<any>} HookedArray */
+/** @typedef {import('../../types.js').Storage} Storage */
+/** @typedef {import('../../types.js').hookArrayOptions<any>} hookArrayOptions */
 
 import { existsSync } from "fs";
 import pLimit from 'p-limit';
@@ -14,34 +17,39 @@ import * as u from "../utils/utils.js";
 /**
  * Creates a hooked array that transforms data on push and handles batching/disk writes
  * @param {Array} arr - Base array to enhance
- * @param {Object} opts - Configuration options
- * @param {Function} opts.hook - Transform function applied to each item
- * @param {string} opts.type - Type identifier for the hook function
- * @param {string} opts.filepath - Base filename for disk writes
- * @param {string} opts.format - Output format ('csv' or 'json')
- * @param {number} opts.concurrency - Max concurrent file operations
- * @param {Context} opts.context - Context object with config, batchSize, etc.
- * @returns {Promise<Array>} Enhanced array with hookPush and flush methods
+ * @param {hookArrayOptions} opts - Configuration options
+ * @returns {Promise<HookedArray>} Enhanced array with hookPush and flush methods
  */
-export async function createHookArray(arr = [], opts = {}) {
+export async function createHookArray(arr = [], opts) {
 	const {
 		hook = a => a,
 		type = "",
 		filepath = "./defaultFile",
 		format = "csv",
 		concurrency = 1,
-		context = {},
+		context = /** @type {Context} */ ({}),
 		...rest
-	} = opts;
+	} = opts || {};
 
 	const FILE_CONN = pLimit(concurrency);
-	const { config = {}, runtime = {} } = context;
+	const { 
+		config = {}, 
+		runtime = { 
+			operations: 0, 
+			eventCount: 0, 
+			userCount: 0, 
+			isBatchMode: false, 
+			verbose: false, 
+			isCLI: false 
+		} 
+	} = context;
 	const BATCH_SIZE = config.batchSize || 1_000_000;
 	const NODE_ENV = process.env.NODE_ENV || "unknown";
 
 	let batch = 0;
 	let writeDir;
 	let isBatchMode = runtime.isBatchMode || false;
+	let isWriting = false; // Prevent concurrent writes
 
 	// Determine write directory
 	const dataFolder = path.resolve("./data");
@@ -77,7 +85,7 @@ export async function createHookArray(arr = [], opts = {}) {
 
 		// Performance optimization: skip hook overhead for passthrough hooks
 		const isPassthroughHook = hook.toString().includes('return record') || hook.length === 1;
-		
+
 		if (isPassthroughHook) {
 			// Fast path for passthrough hooks - no transformation needed
 			if (Array.isArray(item)) {
@@ -112,13 +120,26 @@ export async function createHookArray(arr = [], opts = {}) {
 			}
 		}
 
-		if (arr.length > BATCH_SIZE) {
+		// Check batch size and handle writes synchronously to prevent race conditions
+		if (arr.length > BATCH_SIZE && !isWriting) {
+			isWriting = true; // Lock to prevent concurrent writes
 			isBatchMode = true;
 			runtime.isBatchMode = true; // Update runtime state
 			batch++;
 			const writePath = getWritePath();
-			const writeResult = await FILE_CONN(() => writeToDisk(arr, { writePath }));
-			return writeResult;
+			
+			try {
+				// Create a copy of the data to write
+				const dataToWrite = [...arr];
+				// Clear the array immediately to prevent race conditions
+				arr.length = 0;
+				
+				// Write to disk (this is now synchronous from the perspective of batch management)
+				const writeResult = await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
+				return writeResult;
+			} finally {
+				isWriting = false; // Release the lock
+			}
 		} else {
 			return Promise.resolve(false);
 		}
@@ -129,7 +150,7 @@ export async function createHookArray(arr = [], opts = {}) {
 		let writeResult;
 
 		if (config.verbose) {
-			console.log(`\n\n\twriting ${writePath}\n\n`);
+			console.log(`\n\twriting ${writePath}\n`);
 		}
 
 		switch (format) {
@@ -143,21 +164,33 @@ export async function createHookArray(arr = [], opts = {}) {
 				throw new Error(`format ${format} is not supported`);
 		}
 
-		if (isBatchMode) data.length = 0;
+		// Array clearing now handled in transformThenPush to ensure proper timing
 		return writeResult;
 	}
 
 	async function flush() {
 		if (arr.length > 0) {
-			batch++;
-			const writePath = getWritePath();
-			await FILE_CONN(() => writeToDisk(arr, { writePath }));
-			if (isBatchMode) arr.length = 0; // free up memory for batch mode
+			// Wait for any ongoing writes to complete
+			while (isWriting) {
+				await new Promise(resolve => setTimeout(resolve, 10));
+			}
+			
+			isWriting = true;
+			try {
+				batch++;
+				const writePath = getWritePath();
+				const dataToWrite = [...arr];
+				arr.length = 0; // Clear array after copying data
+				await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
+			} finally {
+				isWriting = false;
+			}
 		}
 	}
 
 	// Enhance the array with our methods
-	const enrichedArray = arr;
+	/** @type {HookedArray} */
+	const enrichedArray = /** @type {any} */ (arr);
 	enrichedArray.hookPush = transformThenPush;
 	enrichedArray.flush = flush;
 	enrichedArray.getWriteDir = getWriteDir;
@@ -181,11 +214,12 @@ export class StorageManager {
 
 	/**
 	 * Initialize all storage containers for the data generation process
-	 * @returns {import('../../types').Storage} Storage containers object
+	 * @returns {Promise<Storage>} Storage containers object
 	 */
 	async initializeContainers() {
 		const { config } = this.context;
 
+		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
@@ -207,7 +241,7 @@ export class StorageManager {
 
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
-				type: "adspend",
+				type: "ad-spend",
 				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
@@ -267,7 +301,7 @@ export class StorageManager {
 					hook: config.hook,
 					type: "lookup",
 					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
-					format: config.format || "csv",
+					format: "csv", // Always force CSV for lookup tables
 					concurrency: config.concurrency || 1,
 					context: this.context
 				});

package/lib/generators/events.js

@@ -102,19 +102,22 @@ export async function makeEvent(
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            eventTemplate.time = dayjs.unix(earliestTime).toISOString();
+            // Apply time shift to move to present day using precomputed value
+            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         } else {
-            eventTemplate.time = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            // Get time from TimeSoup and apply precomputed time shift
+            const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         }
     }
 
     // Add anonymous and session identifiers
     if (anonymousIds.length) {
-        eventTemplate.device_id = chance.pickone(anonymousIds);
+        eventTemplate.device_id = u.pickRandom(anonymousIds);
     }
     
     if (sessionIds.length) {
-        eventTemplate.session_id = chance.pickone(sessionIds);
+        eventTemplate.session_id = u.pickRandom(sessionIds);
     }
 
     // Sometimes add user_id (for attribution modeling)
@@ -127,16 +130,28 @@ export async function makeEvent(
         eventTemplate.user_id = distinct_id;
     }
 
-    // Merge custom properties with super properties    
-	const props = Object.assign({}, chosenEvent.properties, superProps);
-
+    // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
-    for (const key in props) {
-        try {
-            eventTemplate[key] = u.choose(props[key]);
-        } catch (e) {
-            console.error(`error with ${key} in ${chosenEvent.event} event`, e);
-            // Continue processing other properties
+    if (chosenEvent.properties) {
+        for (const key in chosenEvent.properties) {
+            try {
+                eventTemplate[key] = u.choose(chosenEvent.properties[key]);
+            } catch (e) {
+                console.error(`error with ${key} in ${chosenEvent.event} event`, e);
+                // Continue processing other properties
+            }
+        }
+    }
+    
+    // Add super properties (override event properties if needed)
+    if (superProps) {
+        for (const key in superProps) {
+            try {
+                eventTemplate[key] = u.choose(superProps[key]);
+            } catch (e) {
+                console.error(`error with ${key} in super props`, e);
+                // Continue processing other properties
+            }
         }
     }
 
@@ -153,13 +168,21 @@ export async function makeEvent(
 	const tuple = `${eventTemplate.event}-${eventTemplate.time}-${distinctId}`;
     eventTemplate.insert_id = u.quickHash(tuple);
 
-    // Apply time shift to move events to current timeline
-    if (earliestTime) {
-        const timeShift = dayjs().add(2, "day").diff(dayjs.unix(context.FIXED_NOW), "seconds");
-        const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
-        eventTemplate.time = timeShifted;
+    // Call hook if configured (before returning the event)
+    const { hook } = config;
+    if (hook) {
+        const hookedEvent = await hook(eventTemplate, "event", { 
+            user: { distinct_id }, 
+            config 
+        });
+        // If hook returns a modified event, use it; otherwise use original
+        if (hookedEvent && typeof hookedEvent === 'object') {
+            return hookedEvent;
+        }
     }
 
+    // Note: Time shift already applied above during timestamp calculation
+
     return eventTemplate;
 }
 

package/lib/orchestrators/mixpanel-sender.js

@@ -58,9 +58,10 @@ export async function sendToMixpanel(context) {
         epochEnd: dayjs().unix(),
         dryRun: false,
         abridged: false,
-        fixJson: true,
+        fixJson: false,
         showProgress: NODE_ENV === "dev" ? true : false,
-        streamFormat: mpImportFormat
+        streamFormat: mpImportFormat,
+		workers: 35
     };
 
     if (isCLI) commonOpts.showProgress = true;
@@ -134,6 +135,8 @@ export async function sendToMixpanel(context) {
             const imported = await mp({ token, groupKey }, groupProfilesToImport, {
                 recordType: "group",
                 ...commonOpts,
+				groupKey,
+				//dryRun: true
             });
             log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
             importResults.groups.push(imported);