make-mp-data 2.0.18 → 2.0.21

package/entry.js

@@ -27,14 +27,25 @@ import getCliParams from './lib/cli/cli.js';
         const result = await main(finalConfig);        
         console.log(`📊 Generated ${(result.eventCount || 0).toLocaleString()} events for ${(result.userCount || 0).toLocaleString()} users`);
         console.log(`⏱️  Total time: ${result.time?.human || 'unknown'}`);
-        
+		const recordsPerSecond = result.eventCount / result.time.delta * 1000;
+		console.log(`⚡ Records per second: ${recordsPerSecond.toFixed(2)}`);
+		
+		if (result.errors?.length) {
+			console.error(`\n❗ Errors encountered: ${result.errors.length}`);
+			if (cliConfig.verbose) {
+				result.errors.forEach(err => console.error(`   ${err}`));
+			}
+		} else {
+			console.log(`\n🙌 No errors encountered.`);
+		}
+
 		if (result.files?.length) {
             console.log(`📁 Files written: ${result.files.length}`);
             if (cliConfig.verbose) {
                 result.files.forEach(file => console.log(`   ${file}`));
             }
         }
-        console.log(`\n✅ Job completed successfully!`);
+        console.log(`\n👋 Job completed successfully!\n`);
         process.exit(0);
     } catch (error) {
         console.error(`\n❌ Job failed: ${error.message}`);

package/index.js

@@ -47,6 +47,65 @@ global.FIXED_NOW = FIXED_NOW;
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
 global.FIXED_BEGIN = FIXED_BEGIN;
 
+/**
+ * Display configuration summary for CLI mode
+ * @param {Config} config - Validated configuration object
+ */
+function displayConfigurationSummary(config) {
+	console.log('\n📋 Configuration Summary');
+	console.log('─'.repeat(40));
+	
+	// Core parameters
+	console.log(`🎯 Target:      ${config.numUsers?.toLocaleString()} users, ${config.numEvents?.toLocaleString()} events`);
+	console.log(`📅 Timeline:    ${config.numDays} days (${config.seed ? config.seed : 'random seed'})`);
+	console.log(`💾 Output:      ${config.format} format${config.region ? ` (${config.region})` : ''}`);
+	console.log(`⚡ Performance: ${config.concurrency || 'auto'} threads`);
+	
+	// Feature flags
+	const features = [];
+	if (config.hasAnonIds) features.push('anonymous IDs');
+	if (config.hasSessionIds) features.push('session IDs');
+	if (config.alsoInferFunnels) features.push('funnel inference');
+	if (config.makeChart) features.push('chart generation');
+	if (config.writeToDisk) features.push('disk output');
+	
+	if (features.length > 0) {
+		console.log(`🔧 Features:    ${features.join(', ')}`);
+	}
+	
+	// Schema preview
+	if (config.events && config.events.length > 0) {
+		console.log('\n🎭 Event Schema');
+		console.log('─'.repeat(40));
+		const eventNames = config.events.slice(0, 6).map(e => e.event || e).join(', ');
+		const more = config.events.length > 6 ? ` (+${config.events.length - 6} more)` : '';
+		console.log(`📊 Events:      ${eventNames}${more}`);
+	}
+	
+	// Funnels preview
+	if (config.funnels && config.funnels.length > 0) {
+		console.log(`🔄 Funnels:     ${config.funnels.length} funnel${config.funnels.length > 1 ? 's' : ''} configured`);
+		config.funnels.slice(0, 4).forEach((funnel, i) => {
+			if (funnel.sequence) {
+				const arrow = ' → ';
+				const sequence = funnel.sequence.join(arrow);
+				const rate = funnel.conversionRate ? ` (${(funnel.conversionRate * 100).toFixed(0)}% conversion)` : '';
+				console.log(`    ${i + 1}. ${sequence}${rate}`);
+			}
+		});
+		if (config.funnels.length > 4) {
+			console.log(`    ...and ${config.funnels.length - 4} more funnels`);
+		}
+	}
+	
+	// Group analytics
+	if (config.groupKeys && config.groupKeys.length > 0) {
+		const groups = config.groupKeys.map(([key, count]) => `${count} ${key}s`).join(', ');
+		console.log(`👥 Groups:      ${groups}`);
+	}
+	
+	console.log(''); // Extra spacing before generation starts
+}
 
 /**
  * Main data generation function
@@ -81,6 +140,11 @@ async function main(config) {
 	try {
 		// Step 1: Validate and enrich configuration
 		validatedConfig = validateDungeonConfig(config);
+		
+		// Step 1.5: Display configuration summary (CLI mode only)
+		if (isCLI && validatedConfig.verbose) {
+			displayConfigurationSummary(validatedConfig);
+		}
 
 		// Step 2: Create context with validated config
 		const context = createContext(validatedConfig, null, isCLI);
@@ -90,6 +154,8 @@ async function main(config) {
 		const storage = await storageManager.initializeContainers();
 		updateContextWithStorage(context, storage);
 
+		// ! DATA GENERATION STARTS HERE
+
 		// Step 4: Generate ad spend data (if enabled)
 		if (validatedConfig.hasAdSpend) {
 			await generateAdSpendData(context);
@@ -119,6 +185,10 @@ async function main(config) {
 			await makeMirror(context);
 		}
 
+		if (context.config.verbose) console.log(`\n✅ Data generation completed successfully!\n`);
+
+		// ! DATA GENERATION ENDS HERE
+
 		// Step 10: Generate charts (if enabled)
 		if (validatedConfig.makeChart) {
 			await generateCharts(context);
@@ -252,7 +322,8 @@ async function generateLookupTables(context) {
 
 		for (let j = 0; j < entries; j++) {
 			const lookupEntry = await makeProfile(context, attributes, {
-				[key]: `${key}_${j + 1}`
+				id: j + 1 //primary key is always a number so it joins simply with events
+				// [key]: `${key}_${j + 1}` // we don't want to use the lookup name as a prefix here
 			});
 
 			await lookupContainer.hookPush(lookupEntry);
@@ -352,7 +423,7 @@ async function generateCharts(context) {
 	if (config.makeChart && storage.eventData?.length > 0) {
 		const chartPath = typeof config.makeChart === 'string'
 			? config.makeChart
-			: `./charts/${config.simulationName}-timeline.png`;
+			: `./${config.simulationName}-timeline`;
 
 		await generateLineChart(storage.eventData, undefined, chartPath);
 
@@ -395,7 +466,7 @@ async function flushStorageToDisk(storage, config) {
 	await Promise.all(flushPromises);
 
 	if (config.verbose) {
-		console.log('✅ Data flushed to disk successfully');
+		console.log('🙏 Data flushed to disk successfully');
 	}
 }
 

package/lib/cli/cli.js

@@ -100,7 +100,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option('concurrency', {
 			alias: 'conn',
-			default: 500,
+			default: 10,
 			demandOption: false,
 			describe: 'concurrency level for data generation',
 			type: 'number'

package/lib/core/config-validator.js

@@ -3,6 +3,11 @@
  * Extracted from index.js validateDungeonConfig function
  */
 
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+/** @typedef {import('../../types.js').EventConfig} EventConfig */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').Funnel} Funnel */
+
 import dayjs from "dayjs";
 import { makeName } from "ak-tools";
 import * as u from "../utils/utils.js";
@@ -10,8 +15,8 @@ import os from "os";
 
 /**
  * Infers funnels from the provided events
- * @param {Array} events - Array of event configurations
- * @returns {Array} Array of inferred funnel configurations
+ * @param {EventConfig[]} events - Array of event configurations
+ * @returns {Funnel[]} Array of inferred funnel configurations
  */
 function inferFunnels(events) {
     const createdFunnels = [];
@@ -64,8 +69,8 @@ function inferFunnels(events) {
 
 /**
  * Validates and enriches a dungeon configuration object
- * @param {Object} config - Raw configuration object
- * @returns {Object} Validated and enriched configuration
+ * @param {Partial<Dungeon>} config - Raw configuration object
+ * @returns {Dungeon} Validated and enriched configuration
  */
 export function validateDungeonConfig(config) {
     const chance = u.getChance();
@@ -111,9 +116,14 @@ export function validateDungeonConfig(config) {
         alsoInferFunnels = false,
         name = "",
         batchSize = 500_000,
-        concurrency = Math.min(os.cpus().length * 2, 16) // Default to 2x CPU cores, max 16
+        concurrency
     } = config;
 
+    // Set concurrency default only if not provided
+    if (concurrency === undefined || concurrency === null) {
+        concurrency = Math.min(os.cpus().length * 2, 16);
+    }
+
     // Ensure defaults for deep objects
     if (!config.superProps) config.superProps = superProps;
     if (!config.userProps || Object.keys(config?.userProps || {})) config.userProps = userProps;
@@ -133,9 +143,9 @@ export function validateDungeonConfig(config) {
     // Validate events
     if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
     
-    // Convert string events to objects
+    // Convert string events to objects  
     if (typeof events[0] === "string") {
-        events = events.map(e => ({ event: e }));
+        events = events.map(e => ({ event: /** @type {string} */ (e) }));
     }
 
     // Handle funnel inference
@@ -233,6 +243,11 @@ export function validateDungeonConfig(config) {
  * @param {Object} config - Configuration to validate
  * @throws {Error} If required fields are missing
  */
+/**
+ * Validates required configuration parameters
+ * @param {Dungeon} config - Configuration object to validate
+ * @returns {boolean} True if validation passes
+ */
 export function validateRequiredConfig(config) {
     if (!config) {
         throw new Error("Configuration is required");

package/lib/core/context.js

@@ -3,14 +3,14 @@
  * Provides centralized state management and dependency injection
  */
 
-/** @typedef {import('../../types').Dungeon} Dungeon */
-/** @typedef {import('../../types').Storage} Storage */
-/** @typedef {import('../../types').Context} Context */
-/** @typedef {import('../../types').RuntimeState} RuntimeState */
-/** @typedef {import('../../types').Defaults} Defaults */
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+/** @typedef {import('../../types.js').Storage} Storage */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').RuntimeState} RuntimeState */
+/** @typedef {import('../../types.js').Defaults} Defaults */
 
 import dayjs from "dayjs";
-import { campaigns, devices, locations } from '../data/defaults.js';
+import { campaigns, devices, locations } from '../templates/defaults.js';
 import * as u from '../utils/utils.js';
 
 /**
@@ -31,14 +31,23 @@ function createDefaults(config, campaignData) {
 		locations.filter(l => l.country === singleCountry) :
 		locations;
 
+	// PERFORMANCE: Pre-calculate weighted arrays to avoid repeated weighArray calls
+	const weighedLocationsUsers = u.weighArray(locationsUsers);
+	const weighedLocationsEvents = u.weighArray(locationsEvents);
+	const weighedIOSDevices = u.weighArray(devices.iosDevices);
+	const weighedAndroidDevices = u.weighArray(devices.androidDevices);
+	const weighedDesktopDevices = u.weighArray(devices.desktopDevices);
+	const weighedBrowsers = u.weighArray(devices.browsers);
+	const weighedCampaigns = u.weighArray(campaignData);
+
 	return {
-		locationsUsers: () => u.weighArray(locationsUsers),
-		locationsEvents: () => u.weighArray(locationsEvents),
-		iOSDevices: () => u.weighArray(devices.iosDevices),
-		androidDevices: () => u.weighArray(devices.androidDevices),
-		desktopDevices: () => u.weighArray(devices.desktopDevices),
-		browsers: () => u.weighArray(devices.browsers),
-		campaigns: () => u.weighArray(campaignData)
+		locationsUsers: () => weighedLocationsUsers,
+		locationsEvents: () => weighedLocationsEvents,
+		iOSDevices: () => weighedIOSDevices,
+		androidDevices: () => weighedAndroidDevices,
+		desktopDevices: () => weighedDesktopDevices,
+		browsers: () => weighedBrowsers,
+		campaigns: () => weighedCampaigns
 	};
 }
 
@@ -136,17 +145,23 @@ export function createContext(config, storage = null, isCliMode = null) {
 		// Time helper methods
 		getTimeShift() {
 			const actualNow = dayjs().add(2, "day");
-			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
+			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "seconds");
 		},
 
 		getDaysShift() {
 			const actualNow = dayjs().add(2, "day");
-			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "days");
+			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "days");
 		},
 
 		// Time constants (previously globals)
 		FIXED_NOW: global.FIXED_NOW,
-		FIXED_BEGIN: global.FIXED_BEGIN
+		FIXED_BEGIN: global.FIXED_BEGIN,
+		
+		// PERFORMANCE: Pre-calculated time shift (instead of calculating per-event)
+		TIME_SHIFT_SECONDS: (() => {
+			const actualNow = dayjs().add(2, "day");
+			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
+		})(),
 	};
 
 	return context;

package/lib/core/storage.js

@@ -3,7 +3,10 @@
  * Extracted from index.js to eliminate global dependencies
  */
 
-/** @typedef {import('../../types').Context} Context */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').HookedArray<any>} HookedArray */
+/** @typedef {import('../../types.js').Storage} Storage */
+/** @typedef {import('../../types.js').hookArrayOptions<any>} hookArrayOptions */
 
 import { existsSync } from "fs";
 import pLimit from 'p-limit';
@@ -14,25 +17,19 @@ import * as u from "../utils/utils.js";
 /**
  * Creates a hooked array that transforms data on push and handles batching/disk writes
  * @param {Array} arr - Base array to enhance
- * @param {Object} opts - Configuration options
- * @param {Function} opts.hook - Transform function applied to each item
- * @param {string} opts.type - Type identifier for the hook function
- * @param {string} opts.filepath - Base filename for disk writes
- * @param {string} opts.format - Output format ('csv' or 'json')
- * @param {number} opts.concurrency - Max concurrent file operations
- * @param {Context} opts.context - Context object with config, batchSize, etc.
- * @returns {Promise<Array>} Enhanced array with hookPush and flush methods
+ * @param {hookArrayOptions} opts - Configuration options
+ * @returns {Promise<HookedArray>} Enhanced array with hookPush and flush methods
  */
-export async function createHookArray(arr = [], opts = {}) {
+export async function createHookArray(arr = [], opts) {
 	const {
 		hook = a => a,
 		type = "",
 		filepath = "./defaultFile",
 		format = "csv",
 		concurrency = 1,
-		context = {},
+		context = /** @type {Context} */ ({}),
 		...rest
-	} = opts;
+	} = opts || {};
 
 	const FILE_CONN = pLimit(concurrency);
 	const { config = {}, runtime = {} } = context;
@@ -77,7 +74,7 @@ export async function createHookArray(arr = [], opts = {}) {
 
 		// Performance optimization: skip hook overhead for passthrough hooks
 		const isPassthroughHook = hook.toString().includes('return record') || hook.length === 1;
-		
+
 		if (isPassthroughHook) {
 			// Fast path for passthrough hooks - no transformation needed
 			if (Array.isArray(item)) {
@@ -118,6 +115,8 @@ export async function createHookArray(arr = [], opts = {}) {
 			batch++;
 			const writePath = getWritePath();
 			const writeResult = await FILE_CONN(() => writeToDisk(arr, { writePath }));
+			// Ensure array is cleared after successful write
+			arr.length = 0;
 			return writeResult;
 		} else {
 			return Promise.resolve(false);
@@ -129,7 +128,7 @@ export async function createHookArray(arr = [], opts = {}) {
 		let writeResult;
 
 		if (config.verbose) {
-			console.log(`\n\n\twriting ${writePath}\n\n`);
+			console.log(`\n\twriting ${writePath}\n`);
 		}
 
 		switch (format) {
@@ -143,7 +142,7 @@ export async function createHookArray(arr = [], opts = {}) {
 				throw new Error(`format ${format} is not supported`);
 		}
 
-		if (isBatchMode) data.length = 0;
+		// Array clearing now handled in transformThenPush to ensure proper timing
 		return writeResult;
 	}
 
@@ -157,7 +156,8 @@ export async function createHookArray(arr = [], opts = {}) {
 	}
 
 	// Enhance the array with our methods
-	const enrichedArray = arr;
+	/** @type {HookedArray} */
+	const enrichedArray = /** @type {any} */ (arr);
 	enrichedArray.hookPush = transformThenPush;
 	enrichedArray.flush = flush;
 	enrichedArray.getWriteDir = getWriteDir;
@@ -181,11 +181,12 @@ export class StorageManager {
 
 	/**
 	 * Initialize all storage containers for the data generation process
-	 * @returns {import('../../types').Storage} Storage containers object
+	 * @returns {Promise<Storage>} Storage containers object
 	 */
 	async initializeContainers() {
 		const { config } = this.context;
 
+		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
@@ -207,7 +208,7 @@ export class StorageManager {
 
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
-				type: "adspend",
+				type: "ad-spend",
 				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
@@ -267,7 +268,7 @@ export class StorageManager {
 					hook: config.hook,
 					type: "lookup",
 					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
-					format: config.format || "csv",
+					format: "csv", // Always force CSV for lookup tables
 					concurrency: config.concurrency || 1,
 					context: this.context
 				});

package/lib/generators/events.js

@@ -102,19 +102,22 @@ export async function makeEvent(
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            eventTemplate.time = dayjs.unix(earliestTime).toISOString();
+            // Apply time shift to move to present day using precomputed value
+            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         } else {
-            eventTemplate.time = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            // Get time from TimeSoup and apply precomputed time shift
+            const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
         }
     }
 
     // Add anonymous and session identifiers
     if (anonymousIds.length) {
-        eventTemplate.device_id = chance.pickone(anonymousIds);
+        eventTemplate.device_id = u.pickRandom(anonymousIds);
     }
     
     if (sessionIds.length) {
-        eventTemplate.session_id = chance.pickone(sessionIds);
+        eventTemplate.session_id = u.pickRandom(sessionIds);
     }
 
     // Sometimes add user_id (for attribution modeling)
@@ -127,16 +130,28 @@ export async function makeEvent(
         eventTemplate.user_id = distinct_id;
     }
 
-    // Merge custom properties with super properties    
-	const props = Object.assign({}, chosenEvent.properties, superProps);
-
+    // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
-    for (const key in props) {
-        try {
-            eventTemplate[key] = u.choose(props[key]);
-        } catch (e) {
-            console.error(`error with ${key} in ${chosenEvent.event} event`, e);
-            // Continue processing other properties
+    if (chosenEvent.properties) {
+        for (const key in chosenEvent.properties) {
+            try {
+                eventTemplate[key] = u.choose(chosenEvent.properties[key]);
+            } catch (e) {
+                console.error(`error with ${key} in ${chosenEvent.event} event`, e);
+                // Continue processing other properties
+            }
+        }
+    }
+    
+    // Add super properties (override event properties if needed)
+    if (superProps) {
+        for (const key in superProps) {
+            try {
+                eventTemplate[key] = u.choose(superProps[key]);
+            } catch (e) {
+                console.error(`error with ${key} in super props`, e);
+                // Continue processing other properties
+            }
         }
     }
 
@@ -153,13 +168,21 @@ export async function makeEvent(
 	const tuple = `${eventTemplate.event}-${eventTemplate.time}-${distinctId}`;
     eventTemplate.insert_id = u.quickHash(tuple);
 
-    // Apply time shift to move events to current timeline
-    if (earliestTime) {
-        const timeShift = dayjs().add(2, "day").diff(dayjs.unix(context.FIXED_NOW), "seconds");
-        const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
-        eventTemplate.time = timeShifted;
+    // Call hook if configured (before returning the event)
+    const { hook } = config;
+    if (hook) {
+        const hookedEvent = await hook(eventTemplate, "event", { 
+            user: { distinct_id }, 
+            config 
+        });
+        // If hook returns a modified event, use it; otherwise use original
+        if (hookedEvent && typeof hookedEvent === 'object') {
+            return hookedEvent;
+        }
     }
 
+    // Note: Time shift already applied above during timestamp calculation
+
     return eventTemplate;
 }
 

package/lib/orchestrators/worker-manager.js

@@ -3,6 +3,9 @@
  * Handles distributed processing across multiple cloud function workers
  */
 
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+
 import pLimit from 'p-limit';
 import { GoogleAuth } from 'google-auth-library';
 import { timer, uid, sLog } from 'ak-tools';
@@ -138,7 +141,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
     try {
         if (!script) throw new Error("no script");
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const config = eval(script);
         
         if (isReplica) {
@@ -148,7 +151,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
             params.seed = newSeed;
         }
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const optionsYouCantChange = {
             verbose: false
         };