make-mp-data 2.0.17 → 2.0.19

package/dungeons/complex.js

@@ -36,7 +36,7 @@ const config = {
 	makeChart: false,
 
 	batchSize: 500_000,
-	concurrency: 500,
+	concurrency: 10,
 
 	funnels: [],
 	events: [

package/dungeons/sanity.js

@@ -29,6 +29,7 @@ const config = {
 	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	alsoInferFunnels: true, //if true, infer funnels from events
 	makeChart: true,
+	concurrency: 10,
 	funnels: [
 		{
 			sequence: ["qux", "garply", "durtle", "linny", "fonk", "crumn", "yak"],			

package/dungeons/simple.js

@@ -40,6 +40,7 @@ const config = {
 	hasBrowser: true,
 	hasCampaigns: true,
 	isAnonymous: false,
+	concurrency: 10,
 
 
 	events: [
@@ -118,8 +119,8 @@ const config = {
 			}
 		}
 	],
-	superProps: {		
-		currentTheme: weighChoices(["light", "dark", "custom", "light", "dark"]),
+	superProps: {
+		theme: pickAWinner(["light", "dark", "custom", "light", "dark"]),
 	},
 	/*
 	user properties work the same as event properties
@@ -141,10 +142,53 @@ const config = {
 	groupProps: {},
 	lookupTables: [],
 	hook: function (record, type, meta) {
+		if (type === "everything") {
+
+			//custom themers purchase more:
+			const numCustomMode = record.filter(a => a.theme === 'custom').length;
+			const numLightMode = record.filter(a => a.theme === 'light').length;
+			const numDarkMode = record.filter(a => a.theme === 'dark').length;
+			if (numCustomMode > numLightMode || numCustomMode > numDarkMode) {
+				//triple their checkout events
+				const checkoutEvents = record.filter(a => a.event === 'checkout');
+				const newCheckouts = checkoutEvents.map(a => {
+					const randomInt = integer(-48, 48);
+					const newCheckout = {
+						...a,
+						time: dayjs(a.time).add(randomInt, 'hour').toISOString(),
+						event: "checkout",
+						amount: a.amount * 2,
+						coupon: "50%OFF"
+					};
+					return newCheckout;
+				});
+				record.push(...newCheckouts);
+			}
+
+			//users who watch low quality videos churn more:
+			const loQuality = ["480p", "360p", "240p"];
+			const lowQualityWatches = record.filter(a => a.event === 'watch video' && loQuality.includes(a.quality));
+			const highQualityWatches = record.filter(a => a.event === 'watch video' && !loQuality.includes(a.quality));
+			if (lowQualityWatches.length > highQualityWatches.length) {
+				if (flip()) {
+					// find midpoint of records
+					const midpoint = Math.floor(record.length / 2);
+					record = record.slice(0, midpoint);
+
+				}
+			}
+
+		}
+
+
+
 		return record;
 	}
 };
 
+function flip(likelihood = 50) {
+	return chance.bool({ likelihood });
+}
 
 
 export default config;

package/{cli.js → entry.js}

@@ -22,12 +22,13 @@ import getCliParams from './lib/cli/cli.js';
             const simpleConfig = await import('./dungeons/simple.js');
             finalConfig = { ...simpleConfig.default, ...cliConfig };
         }
-
-        const result = await main(finalConfig);
-        
+		
+		
+        const result = await main(finalConfig);        
         console.log(`📊 Generated ${(result.eventCount || 0).toLocaleString()} events for ${(result.userCount || 0).toLocaleString()} users`);
         console.log(`⏱️  Total time: ${result.time?.human || 'unknown'}`);
-        if (result.files?.length) {
+        
+		if (result.files?.length) {
             console.log(`📁 Files written: ${result.files.length}`);
             if (cliConfig.verbose) {
                 result.files.forEach(file => console.log(`   ${file}`));

package/index.js

@@ -60,7 +60,7 @@ async function main(config) {
 	//cli mode check for positional dungeon config
 	const isCLI = config._ && Array.isArray(config._);
 	if (isCLI) {
-		const firstArg = config._.slice().pop()
+		const firstArg = config._.slice().pop();
 		if (firstArg?.endsWith('.js') && existsSync(firstArg)) {
 			if (config.verbose) {
 				console.log(`\n🔍 Loading dungeon config from: ${firstArg}`);
@@ -73,9 +73,10 @@ async function main(config) {
 				throw error;
 			}
 		}
-		
+
 	}
 
+	if (config.verbose) console.log(`\n🔧 Configuring dungeon with seed: ${config.seed}`);
 	let validatedConfig;
 	try {
 		// Step 1: Validate and enrich configuration
@@ -94,6 +95,7 @@ async function main(config) {
 			await generateAdSpendData(context);
 		}
 
+		if (context.config.verbose) console.log(`\n🔄 Starting user and event generation...\n`);
 		// Step 5: Main user and event generation
 		await userLoop(context);
 

package/lib/cli/cli.js

@@ -100,7 +100,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option('concurrency', {
 			alias: 'conn',
-			default: 500,
+			default: 10,
 			demandOption: false,
 			describe: 'concurrency level for data generation',
 			type: 'number'

package/lib/core/config-validator.js

@@ -4,8 +4,9 @@
  */
 
 import dayjs from "dayjs";
-import { makeName, clone } from "ak-tools";
+import { makeName } from "ak-tools";
 import * as u from "../utils/utils.js";
+import os from "os";
 
 /**
  * Infers funnels from the provided events
@@ -34,7 +35,7 @@ function inferFunnels(events) {
     if (firstEvents.length) {
         for (const event of firstEvents) {
             createdFunnels.push({ 
-                ...clone(funnelTemplate), 
+                ...u.deepClone(funnelTemplate), 
                 sequence: [event], 
                 isFirstFunnel: true, 
                 conversionRate: 100 
@@ -43,12 +44,12 @@ function inferFunnels(events) {
     }
 
     // At least one funnel with all usage events
-    createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+    createdFunnels.push({ ...u.deepClone(funnelTemplate), sequence: usageEvents });
 
     // Create random funnels for the rest
     for (let i = 1; i < numFunnelsToCreate; i++) {
         /** @type {import('../../types.js').Funnel} */
-        const funnel = { ...clone(funnelTemplate) };
+        const funnel = { ...u.deepClone(funnelTemplate) };
         funnel.conversionRate = u.integer(25, 75);
         funnel.timeToConvert = u.integer(1, 10);
         funnel.weight = u.integer(1, 10);
@@ -94,7 +95,7 @@ export function validateDungeonConfig(config) {
         token = null,
         region = "US",
         writeToDisk = false,
-        verbose = false,
+        verbose = true,
         makeChart = false,
         soup = {},
         hook = (record) => record,
@@ -110,7 +111,7 @@ export function validateDungeonConfig(config) {
         alsoInferFunnels = false,
         name = "",
         batchSize = 500_000,
-        concurrency = 500
+        concurrency = Math.min(os.cpus().length * 2, 16) // Default to 2x CPU cores, max 16
     } = config;
 
     // Ensure defaults for deep objects

package/lib/core/context.js

@@ -20,26 +20,26 @@ import * as u from '../utils/utils.js';
  * @returns {Defaults} Defaults object with factory functions
  */
 function createDefaults(config, campaignData) {
-    const { singleCountry } = config;
-    
-    // Pre-compute weighted arrays based on configuration
-    const locationsUsers = singleCountry ? 
-        locations.filter(l => l.country === singleCountry) : 
-        locations;
-    
-    const locationsEvents = singleCountry ? 
-        locations.filter(l => l.country === singleCountry) : 
-        locations;
-
-    return {
-        locationsUsers: () => u.weighArray(locationsUsers),
-        locationsEvents: () => u.weighArray(locationsEvents),
-        iOSDevices: () => u.weighArray(devices.iosDevices),
-        androidDevices: () => u.weighArray(devices.androidDevices),
-        desktopDevices: () => u.weighArray(devices.desktopDevices),
-        browsers: () => u.weighArray(devices.browsers),
-        campaigns: () => u.weighArray(campaignData)
-    };
+	const { singleCountry } = config;
+
+	// Pre-compute weighted arrays based on configuration
+	const locationsUsers = singleCountry ?
+		locations.filter(l => l.country === singleCountry) :
+		locations;
+
+	const locationsEvents = singleCountry ?
+		locations.filter(l => l.country === singleCountry) :
+		locations;
+
+	return {
+		locationsUsers: () => u.weighArray(locationsUsers),
+		locationsEvents: () => u.weighArray(locationsEvents),
+		iOSDevices: () => u.weighArray(devices.iosDevices),
+		androidDevices: () => u.weighArray(devices.androidDevices),
+		desktopDevices: () => u.weighArray(devices.desktopDevices),
+		browsers: () => u.weighArray(devices.browsers),
+		campaigns: () => u.weighArray(campaignData)
+	};
 }
 
 /**
@@ -47,14 +47,14 @@ function createDefaults(config, campaignData) {
  * @returns {RuntimeState} Runtime state with counters and flags
  */
 function createRuntimeState() {
-    return {
-        operations: 0,
-        eventCount: 0,
-        userCount: 0,
-        isBatchMode: false,
-        verbose: false,
-        isCLI: false
-    };
+	return {
+		operations: 0,
+		eventCount: 0,
+		userCount: 0,
+		isBatchMode: false,
+		verbose: false,
+		isCLI: false
+	};
 }
 
 /**
@@ -65,90 +65,91 @@ function createRuntimeState() {
  * @returns {Context} Context object containing all state and dependencies
  */
 export function createContext(config, storage = null, isCliMode = null) {
-    // Import campaign data (could be made configurable)
-    const campaignData = campaigns;
-    
-    // Create computed defaults based on config
-    const defaults = createDefaults(config, campaignData);
-    
-    // Create runtime state
-    const runtime = createRuntimeState();
-    
-    // Set runtime flags from config
-    runtime.verbose = config.verbose || false;
-    runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
-    runtime.isCLI = isCliMode !== null ? isCliMode : (process.argv[1]?.endsWith('index.js') || process.argv[1]?.endsWith('cli.js') || false);
-    
-    const context = {
-        config,
-        storage,
-        defaults,
-        campaigns: campaignData,
-        runtime,
-        
-        // Helper methods for updating state
-        incrementOperations() {
-            runtime.operations++;
-        },
-        
-        incrementEvents() {
-            runtime.eventCount++;
-        },
-        
-        incrementUsers() {
-            runtime.userCount++;
-        },
-        
-        setStorage(storageObj) {
-            this.storage = storageObj;
-        },
-        
-        // Getter methods for runtime state
-        getOperations() {
-            return runtime.operations;
-        },
-        
-        getEventCount() {
-            return runtime.eventCount;
-        },
-        
-        getUserCount() {
-            return runtime.userCount;
-        },
-        
-        incrementUserCount() {
-            runtime.userCount++;
-        },
-        
-        incrementEventCount() {
-            runtime.eventCount++;
-        },
-        
-        isBatchMode() {
-            return runtime.isBatchMode;
-        },
-        
-        isCLI() {
-            return runtime.isCLI;
-        },
-        
-        // Time helper methods
-        getTimeShift() {
-            const actualNow = dayjs().add(2, "day");
-            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
-        },
-        
-        getDaysShift() {
-            const actualNow = dayjs().add(2, "day");
-            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "days");
-        },
-        
-        // Time constants (previously globals)
-        FIXED_NOW: global.FIXED_NOW,
-        FIXED_BEGIN: global.FIXED_BEGIN
-    };
-    
-    return context;
+	// Import campaign data (could be made configurable)
+	const campaignData = campaigns;
+
+	// Create computed defaults based on config
+	const defaults = createDefaults(config, campaignData);
+
+	// Create runtime state
+	const runtime = createRuntimeState();
+
+	// Set runtime flags from config
+	runtime.verbose = config.verbose || false;
+	runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
+	runtime.isCLI = isCliMode !== null ? isCliMode : (process.argv[1]?.endsWith('index.js') || process.argv[1]?.endsWith('entry.js') || false);
+	if (runtime.isCLI) runtime.verbose = true; // Always verbose in CLI mode
+
+	const context = {
+		config,
+		storage,
+		defaults,
+		campaigns: campaignData,
+		runtime,
+
+		// Helper methods for updating state
+		incrementOperations() {
+			runtime.operations++;
+		},
+
+		incrementEvents() {
+			runtime.eventCount++;
+		},
+
+		incrementUsers() {
+			runtime.userCount++;
+		},
+
+		setStorage(storageObj) {
+			this.storage = storageObj;
+		},
+
+		// Getter methods for runtime state
+		getOperations() {
+			return runtime.operations;
+		},
+
+		getEventCount() {
+			return runtime.eventCount;
+		},
+
+		getUserCount() {
+			return runtime.userCount;
+		},
+
+		incrementUserCount() {
+			runtime.userCount++;
+		},
+
+		incrementEventCount() {
+			runtime.eventCount++;
+		},
+
+		isBatchMode() {
+			return runtime.isBatchMode;
+		},
+
+		isCLI() {
+			return runtime.isCLI;
+		},
+
+		// Time helper methods
+		getTimeShift() {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
+		},
+
+		getDaysShift() {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "days");
+		},
+
+		// Time constants (previously globals)
+		FIXED_NOW: global.FIXED_NOW,
+		FIXED_BEGIN: global.FIXED_BEGIN
+	};
+
+	return context;
 }
 
 /**
@@ -158,8 +159,8 @@ export function createContext(config, storage = null, isCliMode = null) {
  * @returns {Context} Updated context object
  */
 export function updateContextWithStorage(context, storage) {
-    context.storage = storage;
-    return context;
+	context.storage = storage;
+	return context;
 }
 
 /**
@@ -168,14 +169,14 @@ export function updateContextWithStorage(context, storage) {
  * @throws {Error} If context is missing required properties
  */
 export function validateContext(context) {
-    const required = ['config', 'defaults', 'campaigns', 'runtime'];
-    const missing = required.filter(prop => !context[prop]);
-    
-    if (missing.length > 0) {
-        throw new Error(`Context is missing required properties: ${missing.join(', ')}`);
-    }
-    
-    if (!context.config.numUsers || !context.config.numEvents) {
-        throw new Error('Context config must have numUsers and numEvents');
-    }
+	const required = ['config', 'defaults', 'campaigns', 'runtime'];
+	const missing = required.filter(prop => !context[prop]);
+
+	if (missing.length > 0) {
+		throw new Error(`Context is missing required properties: ${missing.join(', ')}`);
+	}
+
+	if (!context.config.numUsers || !context.config.numEvents) {
+		throw new Error('Context config must have numUsers and numEvents');
+	}
 }

package/lib/core/storage.js

@@ -75,28 +75,41 @@ export async function createHookArray(arr = [], opts = {}) {
 		if (item === null || item === undefined) return false;
 		if (typeof item === 'object' && Object.keys(item).length === 0) return false;
 
-		const allMetaData = { ...rest, ...meta };
-
-		if (Array.isArray(item)) {
-			for (const i of item) {
+		// Performance optimization: skip hook overhead for passthrough hooks
+		const isPassthroughHook = hook.toString().includes('return record') || hook.length === 1;
+		
+		if (isPassthroughHook) {
+			// Fast path for passthrough hooks - no transformation needed
+			if (Array.isArray(item)) {
+				arr.push(...item);
+			} else {
+				arr.push(item);
+			}
+		} else {
+			// Slow path for actual transformation hooks
+			const allMetaData = { ...rest, ...meta };
+
+			if (Array.isArray(item)) {
+				for (const i of item) {
+					try {
+						const enriched = await hook(i, type, allMetaData);
+						if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+						else arr.push(enriched);
+					} catch (e) {
+						console.error(`\n\nyour hook had an error\n\n`, e);
+						arr.push(i);
+					}
+				}
+			} else {
 				try {
-					const enriched = await hook(i, type, allMetaData);
+					const enriched = await hook(item, type, allMetaData);
 					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
 					else arr.push(enriched);
 				} catch (e) {
 					console.error(`\n\nyour hook had an error\n\n`, e);
-					arr.push(i);
+					arr.push(item);
 				}
 			}
-		} else {
-			try {
-				const enriched = await hook(item, type, allMetaData);
-				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-				else arr.push(enriched);
-			} catch (e) {
-				console.error(`\n\nyour hook had an error\n\n`, e);
-				arr.push(item);
-			}
 		}
 
 		if (arr.length > BATCH_SIZE) {

package/lib/generators/adspend.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { md5 } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -74,7 +73,7 @@ function createAdSpendEvent(network, campaign, day, chance) {
 
     // Create unique identifiers
     const id = network.utm_source[0] + '-' + campaign;
-    const uid = md5(id);
+    const uid = u.quickHash(id);
 
     return {
         event: "$ad_spend",

package/lib/generators/events.js

@@ -10,7 +10,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { md5 } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -77,7 +76,7 @@ export async function makeEvent(
 
     // Add default properties based on configuration
     if (hasLocation) {
-        defaultProps.location = u.shuffleArray(defaults.locationsEvents()).pop();
+        defaultProps.location = u.pickRandom(defaults.locationsEvents());
     }
     
     if (hasBrowser) {
@@ -91,13 +90,13 @@ export async function makeEvent(
 
     // Add campaigns with attribution likelihood
     if (hasCampaigns && chance.bool({ likelihood: 25 })) {
-        defaultProps.campaigns = u.shuffleArray(defaults.campaigns()).pop();
+        defaultProps.campaigns = u.pickRandom(defaults.campaigns());
     }
     
     // Select device from pool
     const devices = devicePool.flat();
     if (devices.length) {
-        defaultProps.device = u.shuffleArray(devices).pop();
+        defaultProps.device = u.pickRandom(devices);
     }
 
     // Set event time using TimeSoup for realistic distribution
@@ -128,8 +127,8 @@ export async function makeEvent(
         eventTemplate.user_id = distinct_id;
     }
 
-    // Merge custom properties with super properties
-    const props = { ...chosenEvent.properties, ...superProps };
+    // Merge custom properties with super properties    
+	const props = Object.assign({}, chosenEvent.properties, superProps);
 
     // Add custom properties from event configuration
     for (const key in props) {
@@ -150,7 +149,9 @@ export async function makeEvent(
     addGroupProperties(eventTemplate, groupKeys);
 
     // Generate unique insert_id
-    eventTemplate.insert_id = md5(JSON.stringify(eventTemplate));
+	const distinctId = eventTemplate.user_id || eventTemplate.device_id || eventTemplate.distinct_id || distinct_id;
+	const tuple = `${eventTemplate.event}-${eventTemplate.time}-${distinctId}`;
+    eventTemplate.insert_id = u.quickHash(tuple);
 
     // Apply time shift to move events to current timeline
     if (earliestTime) {

package/lib/generators/funnels.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { clone } from "ak-tools";
 import * as u from "../utils/utils.js";
 import { makeEvent } from "./events.js";
 
@@ -129,7 +128,7 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
     
     return sequence.map((eventName) => {
         const foundEvent = config.events?.find((e) => e.event === eventName);
-        const eventSpec = clone(foundEvent) || { event: eventName, properties: {} };
+        const eventSpec = u.deepClone(foundEvent) || { event: eventName, properties: {} };
         
         // Process event properties
         for (const key in eventSpec.properties) {

package/lib/generators/mirror.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import { clone } from "ak-tools";
 import * as u from "../utils/utils.js";
 
 /**
@@ -44,7 +43,7 @@ export async function makeMirror(context) {
             if (shouldProcessEvent(oldEvent.event, events)) {
                 // Clone event only when needed
                 if (!newEvent) {
-                    newEvent = clone(oldEvent);
+                    newEvent = u.deepClone(oldEvent);
                 }
 
                 // Apply the specified strategy

package/lib/orchestrators/mixpanel-sender.js

@@ -7,7 +7,8 @@
 
 import dayjs from "dayjs";
 import path from "path";
-import { clone, comma, ls, rm } from "ak-tools";
+import { comma, ls, rm } from "ak-tools";
+import * as u from "../utils/utils.js";
 import mp from "mixpanel-import";
 
 /**
@@ -67,10 +68,11 @@ export async function sendToMixpanel(context) {
     // Import events
     if (eventData || isBATCH_MODE) {
         log(`importing events to mixpanel...\n`);
-        let eventDataToImport = clone(eventData);
+        let eventDataToImport = u.deepClone(eventData);
         if (isBATCH_MODE) {
             const writeDir = eventData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             eventDataToImport = files.filter(f => f.includes('-EVENTS-'));
         }
         const imported = await mp(creds, eventDataToImport, {
@@ -84,10 +86,11 @@ export async function sendToMixpanel(context) {
     // Import user profiles
     if (userProfilesData || isBATCH_MODE) {
         log(`importing user profiles to mixpanel...\n`);
-        let userProfilesToImport = clone(userProfilesData);
+        let userProfilesToImport = u.deepClone(userProfilesData);
         if (isBATCH_MODE) {
             const writeDir = userProfilesData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             userProfilesToImport = files.filter(f => f.includes('-USERS-'));
         }
         const imported = await mp(creds, userProfilesToImport, {
@@ -101,10 +104,11 @@ export async function sendToMixpanel(context) {
     // Import ad spend data  
     if (groupEventData || isBATCH_MODE) {
         log(`importing ad spend data to mixpanel...\n`);
-        let adSpendDataToImport = clone(adSpendData);
+        let adSpendDataToImport = u.deepClone(adSpendData);
         if (isBATCH_MODE) {
             const writeDir = adSpendData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             adSpendDataToImport = files.filter(f => f.includes('-AD-SPEND-'));
         }
         const imported = await mp(creds, adSpendDataToImport, {
@@ -120,10 +124,11 @@ export async function sendToMixpanel(context) {
         for (const groupEntity of groupProfilesData) {
             const groupKey = groupEntity?.groupKey;
             log(`importing ${groupKey} profiles to mixpanel...\n`);
-            let groupProfilesToImport = clone(groupEntity);
+            let groupProfilesToImport = u.deepClone(groupEntity);
             if (isBATCH_MODE) {
                 const writeDir = groupEntity.getWriteDir();
                 const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                // @ts-ignore
                 groupProfilesToImport = files.filter(f => f.includes(`-GROUPS-${groupKey}`));
             }
             const imported = await mp({ token, groupKey }, groupProfilesToImport, {
@@ -138,10 +143,11 @@ export async function sendToMixpanel(context) {
     // Import group events
     if (groupEventData || isBATCH_MODE) {
         log(`importing group events to mixpanel...\n`);
-        let groupEventDataToImport = clone(groupEventData);
+        let groupEventDataToImport = u.deepClone(groupEventData);
         if (isBATCH_MODE) {
             const writeDir = groupEventData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS-'));
         }
         const imported = await mp(creds, groupEventDataToImport, {
@@ -160,10 +166,11 @@ export async function sendToMixpanel(context) {
             for (const scdEntity of scdTableData) {
                 const scdKey = scdEntity?.scdKey;
                 log(`importing ${scdKey} SCD data to mixpanel...\n`);
-                let scdDataToImport = clone(scdEntity);
+                let scdDataToImport = u.deepClone(scdEntity);
                 if (isBATCH_MODE) {
                     const writeDir = scdEntity.getWriteDir();
                     const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                    // @ts-ignore
                     scdDataToImport = files.filter(f => f.includes(`-SCD-${scdKey}`));
                 }
 
@@ -198,6 +205,7 @@ export async function sendToMixpanel(context) {
     if (!writeToDisk && isBATCH_MODE) {
         const writeDir = eventData?.getWriteDir() || userProfilesData?.getWriteDir();
         const listDir = await ls(writeDir.split(path.basename(writeDir)).join(""));
+        // @ts-ignore
         const files = listDir.filter(f => 
             f.includes('-EVENTS-') || 
             f.includes('-USERS-') || 

package/lib/orchestrators/user-loop.js

@@ -7,6 +7,7 @@
 
 import dayjs from "dayjs";
 import pLimit from 'p-limit';
+import os from 'os';
 import * as u from "../utils/utils.js";
 import * as t from 'ak-tools';
 import { makeEvent } from "../generators/events.js";
@@ -22,7 +23,7 @@ import { makeSCD } from "../generators/scd.js";
 export async function userLoop(context) {
     const { config, storage, defaults } = context;
     const chance = u.getChance();
-    const concurrency = config?.concurrency || 1;
+    const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
     const USER_CONN = pLimit(concurrency);
     
     const {
@@ -45,8 +46,12 @@ export async function userLoop(context) {
     const avgEvPerUser = numEvents / numUsers;
     const startTime = Date.now();
 
+    // Create batches for parallel processing
+    const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
+    const userPromises = [];
+    
     for (let i = 0; i < numUsers; i++) {
-        await USER_CONN(async () => {
+        const userPromise = USER_CONN(async () => {
             context.incrementUserCount();
             const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
             
@@ -73,7 +78,7 @@ export async function userLoop(context) {
                 : dayjs.unix(global.FIXED_BEGIN);
 
             if (hasLocation) {
-                const location = u.shuffleArray(u.choose(defaults.locationsUsers)).pop();
+                const location = u.pickRandom(u.choose(defaults.locationsUsers));
                 for (const key in location) {
                     user[key] = location[key];
                 }
@@ -83,6 +88,7 @@ export async function userLoop(context) {
             const profile = await makeUserProfile(context, userProps, user);
 
             // SCD creation
+            // @ts-ignore
             const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
             const scdTableKeys = Object.keys(scdUserTables);
 
@@ -129,12 +135,12 @@ export async function userLoop(context) {
                 const timeShift = context.getTimeShift();
                 userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
                 numEventsPreformed += data.length;
-                usersEvents.push(...data);
+                usersEvents = usersEvents.concat(data);
                 
                 if (!userConverted) {
-                    if (verbose) {
-                        u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-                    }
+                    // if (verbose) {
+                    //     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+                    // }
                     return;
                 }
             } else {
@@ -146,11 +152,11 @@ export async function userLoop(context) {
                     const currentFunnel = chance.pickone(usageFunnels);
                     const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
                     numEventsPreformed += data.length;
-                    usersEvents.push(...data);
+                    usersEvents = usersEvents.concat(data);
                 } else {
                     const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
                     numEventsPreformed++;
-                    usersEvents.push(data);
+                    usersEvents = usersEvents.concat(data);
                 }
             }
 
@@ -187,8 +193,13 @@ export async function userLoop(context) {
             await eventData.hookPush(usersEvents, { profile });
 
             if (verbose) {
-                u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+                // u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
             }
         });
+        
+        userPromises.push(userPromise);
     }
+    
+    // Wait for all users to complete
+    await Promise.all(userPromises);
 }

package/lib/utils/utils.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import Chance from 'chance';
 import readline from 'readline';
-import { comma, uid, clone } from 'ak-tools';
+import { comma, uid} from 'ak-tools';
 import { spawn } from 'child_process';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
@@ -187,9 +187,26 @@ function choose(value) {
 	const chance = getChance();
 
 	try {
-		// Keep resolving the value if it's a function
+		// Keep resolving the value if it's a function (with caching)
 		while (typeof value === 'function') {
-			value = value();
+			const funcString = value.toString();
+
+			// Check cache for weighted array functions
+			if (typeof global.weightedArrayCache === 'undefined') {
+				global.weightedArrayCache = new Map();
+			}
+
+			if (global.weightedArrayCache.has(funcString)) {
+				value = global.weightedArrayCache.get(funcString);
+				break;
+			}
+
+			const result = value();
+			if (Array.isArray(result) && result.length > 10) {
+				// Cache large arrays (likely weighted arrays)
+				global.weightedArrayCache.set(funcString, result);
+			}
+			value = result;
 		}
 
 		if (Array.isArray(value) && value.length === 0) {
@@ -666,6 +683,21 @@ function pickAWinner(items, mostChosenIndex) {
 	};
 }
 
+function quickHash(str, seed = 0) {
+    let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
+    for (let i = 0, ch; i < str.length; i++) {
+        ch = str.charCodeAt(i);
+        h1 = Math.imul(h1 ^ ch, 2654435761);
+        h2 = Math.imul(h2 ^ ch, 1597334677);
+    }
+    h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+    h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+    h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+    h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+
+    return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
+};
+
 /*
 ----
 SHUFFLERS
@@ -677,11 +709,20 @@ function shuffleArray(array) {
 	const chance = getChance();
 	for (let i = array.length - 1; i > 0; i--) {
 		const j = chance.integer({ min: 0, max: i });
-		[array[i], array[j]] = [array[j], array[i]];
+		const temp = array[i];
+		array[i] = array[j];
+		array[j] = temp;
 	}
 	return array;
 }
 
+function pickRandom(array) {
+    const chance = getChance();
+    if (!array || array.length === 0) return undefined;
+    const randomIndex = chance.integer({ min: 0, max: array.length - 1 });
+    return array[randomIndex];
+}
+
 function shuffleExceptFirst(array) {
 	if (array.length <= 1) return array;
 	const restShuffled = shuffleArray(array.slice(1));
@@ -877,16 +918,26 @@ function buildFileNames(config) {
  * @param  {[string, number][]} arrayOfArrays
  */
 function progress(arrayOfArrays) {
-	readline.cursorTo(process.stdout, 0);
-	let message = "";
-	for (const status of arrayOfArrays) {
-		const [thing, p] = status;
-		message += `${thing} processed: ${comma(p)}\t\t`;
-	}
-
-	process.stdout.write(message);
-};
-
+    const terminalWidth = process.stdout.columns || 120;
+    
+    // Clear the entire line
+    readline.cursorTo(process.stdout, 0);
+    readline.clearLine(process.stdout, 0);
+    
+    // Build message with better formatting
+    const items = arrayOfArrays.map(([thing, p]) => {
+        return `${thing}: ${comma(p)}`;
+    });
+    
+    const message = items.join(' │ ');
+    
+    // Ensure we don't exceed terminal width
+    const finalMessage = message.length > terminalWidth 
+        ? message.substring(0, terminalWidth - 3) + '...'
+        : message.padEnd(terminalWidth, ' ');
+    
+    process.stdout.write(finalMessage);
+}
 
 function openFinder(path, callback) {
 	path = path || '/';
@@ -906,7 +957,6 @@ function getUniqueKeys(data) {
 };
 
 
-
 /*
 ----
 CORE
@@ -953,7 +1003,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	let totalRange = latestTime - earliestTime;
 	if (totalRange <= 0 || earliestTime > latestTime) {
 		//just flip earliest and latest
-		let tempEarly = latestTime
+		let tempEarly = latestTime;
 		let tempLate = earliestTime;
 		earliestTime = tempEarly;
 		latestTime = tempLate;
@@ -967,23 +1017,22 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	const chunkEnd = chunkStart + chunkSize;
 	const chunkMid = (chunkStart + chunkEnd) / 2;
 
-	// Generate a single timestamp within this chunk using a normal distribution centered at chunkMid
-	let offset;
-	let iterations = 0;
-	let isValidTime = false;
-	do {
-		iterations++;
-		soupHits++;
-		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
-		isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
-		if (iterations > 25000) {
-			if (process.env?.NODE_ENV === 'dev') debugger;
-			throw `${iterations} iterations... exceeded`;
-		}
-	} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
+	// Optimized timestamp generation - clamp to valid range instead of looping
+	const maxDeviation = chunkSize / deviation;
+	let offset = chance.normal({ mean: mean, dev: maxDeviation });
+
+	// Clamp to chunk boundaries to prevent infinite loops
+	const proposedTime = chunkMid + offset;
+	const clampedTime = Math.max(chunkStart, Math.min(chunkEnd, proposedTime));
+
+	// Ensure it's within the overall valid range
+	const finalTime = Math.max(earliestTime, Math.min(latestTime, clampedTime));
+
+	// Update soup hits counter (keep for compatibility)
+	soupHits++;
 
 	try {
-		return dayjs.unix(chunkMid + offset).toISOString();
+		return dayjs.unix(finalTime).toISOString();
 	}
 
 	catch (e) {
@@ -994,8 +1043,6 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 }
 
 
-
-
 /**
  * @param {string} userId
  * @param  {number} bornDaysAgo=30
@@ -1151,6 +1198,61 @@ function generateEmoji(max = 10, array = false) {
 	};
 };
 
+function deepClone(thing, opts) {
+	// Handle primitives first (most common case)
+	if (thing === null || thing === undefined) return thing;
+
+	const type = typeof thing;
+	if (type !== 'object' && type !== 'function') {
+		if (type === 'symbol') {
+			return Symbol(thing.description);
+		}
+		return thing;
+	}
+
+	// Handle arrays (common case)
+	if (Array.isArray(thing)) {
+		const result = new Array(thing.length);
+		for (let i = 0; i < thing.length; i++) {
+			result[i] = deepClone(thing[i], opts);
+		}
+		return result;
+	}
+
+	// Handle other object types
+	if (thing instanceof Date) return new Date(thing.getTime());
+	if (thing instanceof RegExp) return new RegExp(thing.source, thing.flags);
+	if (thing instanceof Function) {
+		return opts && opts.newFns ?
+			new Function('return ' + thing.toString())() :
+			thing;
+	}
+
+	// Handle plain objects
+	if (thing.constructor === Object) {
+		const newObject = {};
+		const keys = Object.keys(thing);
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			newObject[key] = deepClone(thing[key], opts);
+		}
+		return newObject;
+	}
+
+	// Handle other object types
+	try {
+		return new thing.constructor(thing);
+	} catch (e) {
+		// Fallback for objects that can't be constructed this way
+		const newObject = Object.create(Object.getPrototypeOf(thing));
+		const keys = Object.keys(thing);
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			newObject[key] = deepClone(thing[key], opts);
+		}
+		return newObject;
+	}
+};
 
 
 export {
@@ -1159,13 +1261,14 @@ export {
 	dates,
 	day,
 	choose,
+	pickRandom,
 	exhaust,
 	integer,
 	TimeSoup,
 	companyName,
 	generateEmoji,
 	hasSameKeys as haveSameKeys,
-
+	deepClone,
 	initChance,
 	getChance,
 
@@ -1182,6 +1285,7 @@ export {
 	getUniqueKeys,
 	person,
 	pickAWinner,
+	quickHash,
 	weighArray,
 	weighFunnels,
 	validateEventConfig,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "2.0.17",
+	"version": "2.0.19",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
@@ -22,13 +22,13 @@
 		"post": "npm publish",
 		"deps": "./scripts/update-deps.sh",
 		"test": "NODE_ENV=test vitest run",
-		"test:watch": "NODE_ENV=test vitest",
-		"test:ui": "NODE_ENV=test vitest --ui",
 		"coverage": "vitest run --coverage && open ./coverage/index.html",
 		"typecheck": "tsc --noEmit",
 		"new:dungeon": "./scripts/new-dungeon.sh",
 		"new:project": "node ./scripts/new-project.mjs",
-		"exp:benchmark": "node --no-warnings --experimental-vm-modules ./tests/benchmark/concurrency.mjs",
+		"exp:benchmark": "node ./tests/benchmark/concurrency.mjs",
+		"benchmark:phase1": "node ./tests/benchmark/phase1-performance.mjs",
+		"test:quick": "node ./tests/benchmark/quick-test.mjs",
 		"exp:soup": "node ./tests/testSoup.mjs",
 		"func:local": "functions-framework --target=entry",
 		"func:deploy": "./scripts/deploy.sh"
@@ -38,7 +38,7 @@
 		"url": "git+https://github.com/ak--47/make-mp-data.git"
 	},
 	"bin": {
-		"make-mp-data": "./cli.js"
+		"make-mp-data": "./entry.js"
 	},
 	"keywords": [
 		"mixpanel",