make-mp-data 2.0.0 → 2.0.2

package/lib/core/context.js

@@ -3,11 +3,11 @@
  * Provides centralized state management and dependency injection
  */
 
-/** @typedef {import('../../types').Dungeon} Dungeon */
-/** @typedef {import('../../types').Storage} Storage */
-/** @typedef {import('../../types').Context} Context */
-/** @typedef {import('../../types').RuntimeState} RuntimeState */
-/** @typedef {import('../../types').Defaults} Defaults */
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+/** @typedef {import('../../types.js').Storage} Storage */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').RuntimeState} RuntimeState */
+/** @typedef {import('../../types.js').Defaults} Defaults */
 
 import dayjs from "dayjs";
 import { campaigns, devices, locations } from '../data/defaults.js';
@@ -20,26 +20,35 @@ import * as u from '../utils/utils.js';
  * @returns {Defaults} Defaults object with factory functions
  */
 function createDefaults(config, campaignData) {
-    const { singleCountry } = config;
-    
-    // Pre-compute weighted arrays based on configuration
-    const locationsUsers = singleCountry ? 
-        locations.filter(l => l.country === singleCountry) : 
-        locations;
-    
-    const locationsEvents = singleCountry ? 
-        locations.filter(l => l.country === singleCountry) : 
-        locations;
-
-    return {
-        locationsUsers: () => u.weighArray(locationsUsers),
-        locationsEvents: () => u.weighArray(locationsEvents),
-        iOSDevices: () => u.weighArray(devices.iosDevices),
-        androidDevices: () => u.weighArray(devices.androidDevices),
-        desktopDevices: () => u.weighArray(devices.desktopDevices),
-        browsers: () => u.weighArray(devices.browsers),
-        campaigns: () => u.weighArray(campaignData)
-    };
+	const { singleCountry } = config;
+
+	// Pre-compute weighted arrays based on configuration
+	const locationsUsers = singleCountry ?
+		locations.filter(l => l.country === singleCountry) :
+		locations;
+
+	const locationsEvents = singleCountry ?
+		locations.filter(l => l.country === singleCountry) :
+		locations;
+
+	// PERFORMANCE: Pre-calculate weighted arrays to avoid repeated weighArray calls
+	const weighedLocationsUsers = u.weighArray(locationsUsers);
+	const weighedLocationsEvents = u.weighArray(locationsEvents);
+	const weighedIOSDevices = u.weighArray(devices.iosDevices);
+	const weighedAndroidDevices = u.weighArray(devices.androidDevices);
+	const weighedDesktopDevices = u.weighArray(devices.desktopDevices);
+	const weighedBrowsers = u.weighArray(devices.browsers);
+	const weighedCampaigns = u.weighArray(campaignData);
+
+	return {
+		locationsUsers: () => weighedLocationsUsers,
+		locationsEvents: () => weighedLocationsEvents,
+		iOSDevices: () => weighedIOSDevices,
+		androidDevices: () => weighedAndroidDevices,
+		desktopDevices: () => weighedDesktopDevices,
+		browsers: () => weighedBrowsers,
+		campaigns: () => weighedCampaigns
+	};
 }
 
 /**
@@ -47,107 +56,115 @@ function createDefaults(config, campaignData) {
  * @returns {RuntimeState} Runtime state with counters and flags
  */
 function createRuntimeState() {
-    return {
-        operations: 0,
-        eventCount: 0,
-        userCount: 0,
-        isBatchMode: false,
-        verbose: false,
-        isCLI: false
-    };
+	return {
+		operations: 0,
+		eventCount: 0,
+		userCount: 0,
+		isBatchMode: false,
+		verbose: false,
+		isCLI: false
+	};
 }
 
 /**
  * Context factory that creates a complete context object for data generation
  * @param {Dungeon} config - Validated configuration object
  * @param {Storage|null} storage - Storage containers (optional, can be set later)
+ * @param {boolean} [isCliMode] - Whether running in CLI mode (optional, will detect if not provided)
  * @returns {Context} Context object containing all state and dependencies
  */
-export function createContext(config, storage = null) {
-    // Import campaign data (could be made configurable)
-    const campaignData = campaigns;
-    
-    // Create computed defaults based on config
-    const defaults = createDefaults(config, campaignData);
-    
-    // Create runtime state
-    const runtime = createRuntimeState();
-    
-    // Set runtime flags from config
-    runtime.verbose = config.verbose || false;
-    runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
-    runtime.isCLI = process.argv[1].endsWith('index.js') || process.argv[1].endsWith('cli.js');
-    
-    const context = {
-        config,
-        storage,
-        defaults,
-        campaigns: campaignData,
-        runtime,
-        
-        // Helper methods for updating state
-        incrementOperations() {
-            runtime.operations++;
-        },
-        
-        incrementEvents() {
-            runtime.eventCount++;
-        },
-        
-        incrementUsers() {
-            runtime.userCount++;
-        },
-        
-        setStorage(storageObj) {
-            this.storage = storageObj;
-        },
-        
-        // Getter methods for runtime state
-        getOperations() {
-            return runtime.operations;
-        },
-        
-        getEventCount() {
-            return runtime.eventCount;
-        },
-        
-        getUserCount() {
-            return runtime.userCount;
-        },
-        
-        incrementUserCount() {
-            runtime.userCount++;
-        },
-        
-        incrementEventCount() {
-            runtime.eventCount++;
-        },
-        
-        isBatchMode() {
-            return runtime.isBatchMode;
-        },
-        
-        isCLI() {
-            return runtime.isCLI;
-        },
-        
-        // Time helper methods
-        getTimeShift() {
-            const actualNow = dayjs().add(2, "day");
-            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
-        },
-        
-        getDaysShift() {
-            const actualNow = dayjs().add(2, "day");
-            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "days");
-        },
-        
-        // Time constants (previously globals)
-        FIXED_NOW: global.FIXED_NOW,
-        FIXED_BEGIN: global.FIXED_BEGIN
-    };
-    
-    return context;
+export function createContext(config, storage = null, isCliMode = null) {
+	// Import campaign data (could be made configurable)
+	const campaignData = campaigns;
+
+	// Create computed defaults based on config
+	const defaults = createDefaults(config, campaignData);
+
+	// Create runtime state
+	const runtime = createRuntimeState();
+
+	// Set runtime flags from config
+	runtime.verbose = config.verbose || false;
+	runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
+	runtime.isCLI = isCliMode !== null ? isCliMode : (process.argv[1]?.endsWith('index.js') || process.argv[1]?.endsWith('entry.js') || false);
+	if (runtime.isCLI) runtime.verbose = true; // Always verbose in CLI mode
+
+	const context = {
+		config,
+		storage,
+		defaults,
+		campaigns: campaignData,
+		runtime,
+
+		// Helper methods for updating state
+		incrementOperations() {
+			runtime.operations++;
+		},
+
+		incrementEvents() {
+			runtime.eventCount++;
+		},
+
+		incrementUsers() {
+			runtime.userCount++;
+		},
+
+		setStorage(storageObj) {
+			this.storage = storageObj;
+		},
+
+		// Getter methods for runtime state
+		getOperations() {
+			return runtime.operations;
+		},
+
+		getEventCount() {
+			return runtime.eventCount;
+		},
+
+		getUserCount() {
+			return runtime.userCount;
+		},
+
+		incrementUserCount() {
+			runtime.userCount++;
+		},
+
+		incrementEventCount() {
+			runtime.eventCount++;
+		},
+
+		isBatchMode() {
+			return runtime.isBatchMode;
+		},
+
+		isCLI() {
+			return runtime.isCLI;
+		},
+
+		// Time helper methods
+		getTimeShift() {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "seconds");
+		},
+
+		getDaysShift() {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "days");
+		},
+
+		// Time constants (previously globals)
+		FIXED_NOW: global.FIXED_NOW,
+		FIXED_BEGIN: global.FIXED_BEGIN,
+		
+		// PERFORMANCE: Pre-calculated time shift (instead of calculating per-event)
+		TIME_SHIFT_SECONDS: (() => {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
+		})(),
+	};
+
+	return context;
 }
 
 /**
@@ -157,8 +174,8 @@ export function createContext(config, storage = null) {
  * @returns {Context} Updated context object
  */
 export function updateContextWithStorage(context, storage) {
-    context.storage = storage;
-    return context;
+	context.storage = storage;
+	return context;
 }
 
 /**
@@ -167,14 +184,14 @@ export function updateContextWithStorage(context, storage) {
  * @throws {Error} If context is missing required properties
  */
 export function validateContext(context) {
-    const required = ['config', 'defaults', 'campaigns', 'runtime'];
-    const missing = required.filter(prop => !context[prop]);
-    
-    if (missing.length > 0) {
-        throw new Error(`Context is missing required properties: ${missing.join(', ')}`);
-    }
-    
-    if (!context.config.numUsers || !context.config.numEvents) {
-        throw new Error('Context config must have numUsers and numEvents');
-    }
+	const required = ['config', 'defaults', 'campaigns', 'runtime'];
+	const missing = required.filter(prop => !context[prop]);
+
+	if (missing.length > 0) {
+		throw new Error(`Context is missing required properties: ${missing.join(', ')}`);
+	}
+
+	if (!context.config.numUsers || !context.config.numEvents) {
+		throw new Error('Context config must have numUsers and numEvents');
+	}
 }

package/lib/core/storage.js

@@ -3,7 +3,10 @@
  * Extracted from index.js to eliminate global dependencies
  */
 
-/** @typedef {import('../../types').Context} Context */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').HookedArray<any>} HookedArray */
+/** @typedef {import('../../types.js').Storage} Storage */
+/** @typedef {import('../../types.js').hookArrayOptions<any>} hookArrayOptions */
 
 import { existsSync } from "fs";
 import pLimit from 'p-limit';
@@ -14,25 +17,19 @@ import * as u from "../utils/utils.js";
 /**
  * Creates a hooked array that transforms data on push and handles batching/disk writes
  * @param {Array} arr - Base array to enhance
- * @param {Object} opts - Configuration options
- * @param {Function} opts.hook - Transform function applied to each item
- * @param {string} opts.type - Type identifier for the hook function
- * @param {string} opts.filepath - Base filename for disk writes
- * @param {string} opts.format - Output format ('csv' or 'json')
- * @param {number} opts.concurrency - Max concurrent file operations
- * @param {Context} opts.context - Context object with config, batchSize, etc.
- * @returns {Promise<Array>} Enhanced array with hookPush and flush methods
+ * @param {hookArrayOptions} opts - Configuration options
+ * @returns {Promise<HookedArray>} Enhanced array with hookPush and flush methods
  */
-export async function createHookArray(arr = [], opts = {}) {
+export async function createHookArray(arr = [], opts) {
 	const {
 		hook = a => a,
 		type = "",
 		filepath = "./defaultFile",
 		format = "csv",
 		concurrency = 1,
-		context = {},
+		context = /** @type {Context} */ ({}),
 		...rest
-	} = opts;
+	} = opts || {};
 
 	const FILE_CONN = pLimit(concurrency);
 	const { config = {}, runtime = {} } = context;
@@ -75,28 +72,41 @@ export async function createHookArray(arr = [], opts = {}) {
 		if (item === null || item === undefined) return false;
 		if (typeof item === 'object' && Object.keys(item).length === 0) return false;
 
-		const allMetaData = { ...rest, ...meta };
+		// Performance optimization: skip hook overhead for passthrough hooks
+		const isPassthroughHook = hook.toString().includes('return record') || hook.length === 1;
 
-		if (Array.isArray(item)) {
-			for (const i of item) {
+		if (isPassthroughHook) {
+			// Fast path for passthrough hooks - no transformation needed
+			if (Array.isArray(item)) {
+				arr.push(...item);
+			} else {
+				arr.push(item);
+			}
+		} else {
+			// Slow path for actual transformation hooks
+			const allMetaData = { ...rest, ...meta };
+
+			if (Array.isArray(item)) {
+				for (const i of item) {
+					try {
+						const enriched = await hook(i, type, allMetaData);
+						if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+						else arr.push(enriched);
+					} catch (e) {
+						console.error(`\n\nyour hook had an error\n\n`, e);
+						arr.push(i);
+					}
+				}
+			} else {
 				try {
-					const enriched = await hook(i, type, allMetaData);
+					const enriched = await hook(item, type, allMetaData);
 					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
 					else arr.push(enriched);
 				} catch (e) {
 					console.error(`\n\nyour hook had an error\n\n`, e);
-					arr.push(i);
+					arr.push(item);
 				}
 			}
-		} else {
-			try {
-				const enriched = await hook(item, type, allMetaData);
-				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-				else arr.push(enriched);
-			} catch (e) {
-				console.error(`\n\nyour hook had an error\n\n`, e);
-				arr.push(item);
-			}
 		}
 
 		if (arr.length > BATCH_SIZE) {
@@ -105,6 +115,8 @@ export async function createHookArray(arr = [], opts = {}) {
 			batch++;
 			const writePath = getWritePath();
 			const writeResult = await FILE_CONN(() => writeToDisk(arr, { writePath }));
+			// Ensure array is cleared after successful write
+			arr.length = 0;
 			return writeResult;
 		} else {
 			return Promise.resolve(false);
@@ -116,7 +128,7 @@ export async function createHookArray(arr = [], opts = {}) {
 		let writeResult;
 
 		if (config.verbose) {
-			console.log(`\n\n\twriting ${writePath}\n\n`);
+			console.log(`\n\twriting ${writePath}\n`);
 		}
 
 		switch (format) {
@@ -130,7 +142,7 @@ export async function createHookArray(arr = [], opts = {}) {
 				throw new Error(`format ${format} is not supported`);
 		}
 
-		if (isBatchMode) data.length = 0;
+		// Array clearing now handled in transformThenPush to ensure proper timing
 		return writeResult;
 	}
 
@@ -144,7 +156,8 @@ export async function createHookArray(arr = [], opts = {}) {
 	}
 
 	// Enhance the array with our methods
-	const enrichedArray = arr;
+	/** @type {HookedArray} */
+	const enrichedArray = /** @type {any} */ (arr);
 	enrichedArray.hookPush = transformThenPush;
 	enrichedArray.flush = flush;
 	enrichedArray.getWriteDir = getWriteDir;
@@ -168,11 +181,12 @@ export class StorageManager {
 
 	/**
 	 * Initialize all storage containers for the data generation process
-	 * @returns {import('../../types').Storage} Storage containers object
+	 * @returns {Promise<Storage>} Storage containers object
 	 */
 	async initializeContainers() {
 		const { config } = this.context;
 
+		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
@@ -194,7 +208,7 @@ export class StorageManager {
 
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
-				type: "adspend",
+				type: "ad-spend",
 				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,

package/lib/data/defaults.js

@@ -4,8 +4,8 @@
  * @fileoverview Contains default values for campaigns, devices, locations, and domains
  */
 
-/** @typedef {import('../../types.d.ts').main.Dungeon} Config */
-/** @typedef {import('../../types.d.ts').main.ValueValid} ValueValid */
+/** @typedef {import('../../types.js').Dungeon} Config */
+/** @typedef {import('../../types.js').ValueValid} ValueValid */
 
 //? https://docs.mixpanel.com/docs/data-structure/property-reference#default-properties
 

package/lib/generators/adspend.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { md5 } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -74,7 +73,7 @@ function createAdSpendEvent(network, campaign, day, chance) {
 
     // Create unique identifiers
     const id = network.utm_source[0] + '-' + campaign;
-    const uid = md5(id);
+    const uid = u.quickHash(id);
 
     return {
         event: "$ad_spend",

package/lib/generators/events.js

@@ -10,7 +10,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { md5 } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -77,7 +76,7 @@ export async function makeEvent(
 
     // Add default properties based on configuration
     if (hasLocation) {
-        defaultProps.location = u.shuffleArray(defaults.locationsEvents()).pop();
+        defaultProps.location = u.pickRandom(defaults.locationsEvents());
     }
     
     if (hasBrowser) {
@@ -91,31 +90,34 @@ export async function makeEvent(
 
     // Add campaigns with attribution likelihood
     if (hasCampaigns && chance.bool({ likelihood: 25 })) {
-        defaultProps.campaigns = u.shuffleArray(defaults.campaigns()).pop();
+        defaultProps.campaigns = u.pickRandom(defaults.campaigns());
     }
     
     // Select device from pool
     const devices = devicePool.flat();
     if (devices.length) {
-        defaultProps.device = u.shuffleArray(devices).pop();
+        defaultProps.device = u.pickRandom(devices);
     }
 
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            eventTemplate.time = dayjs.unix(earliestTime).toISOString();
+            // Apply time shift to move to present day using precomputed value
+            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         } else {
-            eventTemplate.time = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            // Get time from TimeSoup and apply precomputed time shift
+            const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         }
     }
 
     // Add anonymous and session identifiers
     if (anonymousIds.length) {
-        eventTemplate.device_id = chance.pickone(anonymousIds);
+        eventTemplate.device_id = u.pickRandom(anonymousIds);
     }
     
     if (sessionIds.length) {
-        eventTemplate.session_id = chance.pickone(sessionIds);
+        eventTemplate.session_id = u.pickRandom(sessionIds);
     }
 
     // Sometimes add user_id (for attribution modeling)
@@ -128,16 +130,28 @@ export async function makeEvent(
         eventTemplate.user_id = distinct_id;
     }
 
-    // Merge custom properties with super properties
-    const props = { ...chosenEvent.properties, ...superProps };
-
+    // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
-    for (const key in props) {
-        try {
-            eventTemplate[key] = u.choose(props[key]);
-        } catch (e) {
-            console.error(`error with ${key} in ${chosenEvent.event} event`, e);
-            // Continue processing other properties
+    if (chosenEvent.properties) {
+        for (const key in chosenEvent.properties) {
+            try {
+                eventTemplate[key] = u.choose(chosenEvent.properties[key]);
+            } catch (e) {
+                console.error(`error with ${key} in ${chosenEvent.event} event`, e);
+                // Continue processing other properties
+            }
+        }
+    }
+    
+    // Add super properties (override event properties if needed)
+    if (superProps) {
+        for (const key in superProps) {
+            try {
+                eventTemplate[key] = u.choose(superProps[key]);
+            } catch (e) {
+                console.error(`error with ${key} in super props`, e);
+                // Continue processing other properties
+            }
         }
     }
 
@@ -150,14 +164,11 @@ export async function makeEvent(
     addGroupProperties(eventTemplate, groupKeys);
 
     // Generate unique insert_id
-    eventTemplate.insert_id = md5(JSON.stringify(eventTemplate));
+	const distinctId = eventTemplate.user_id || eventTemplate.device_id || eventTemplate.distinct_id || distinct_id;
+	const tuple = `${eventTemplate.event}-${eventTemplate.time}-${distinctId}`;
+    eventTemplate.insert_id = u.quickHash(tuple);
 
-    // Apply time shift to move events to current timeline
-    if (earliestTime) {
-        const timeShift = dayjs().add(2, "day").diff(dayjs.unix(context.FIXED_NOW), "seconds");
-        const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
-        eventTemplate.time = timeShifted;
-    }
+    // Note: Time shift already applied above during timestamp calculation
 
     return eventTemplate;
 }

package/lib/generators/funnels.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { clone } from "ak-tools";
 import * as u from "../utils/utils.js";
 import { makeEvent } from "./events.js";
 
@@ -129,7 +128,7 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
     
     return sequence.map((eventName) => {
         const foundEvent = config.events?.find((e) => e.event === eventName);
-        const eventSpec = clone(foundEvent) || { event: eventName, properties: {} };
+        const eventSpec = u.deepClone(foundEvent) || { event: eventName, properties: {} };
         
         // Process event properties
         for (const key in eventSpec.properties) {

package/lib/generators/mirror.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { clone } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -44,7 +43,7 @@ export async function makeMirror(context) {
             if (shouldProcessEvent(oldEvent.event, events)) {
                 // Clone event only when needed
                 if (!newEvent) {
-                    newEvent = clone(oldEvent);
+                    newEvent = u.deepClone(oldEvent);
                 }
 
                 // Apply the specified strategy