make-mp-data 2.0.22 → 2.1.0

package/dungeons/userAgent.js

@@ -14,7 +14,7 @@ const days = 30;
 
 /** @type {Config} */
 const config = {
-	token: "a854e82c2c91ada31c502139d99e8e0d",
+	token: "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 100,
@@ -33,7 +33,6 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
-	makeChart: false,
 
 	batchSize: 500_000,
 	concurrency: 500,

package/entry.js

@@ -27,6 +27,7 @@ import getCliParams from './lib/cli/cli.js';
             const simpleConfig = await import('./dungeons/simple.js');
             finalConfig = { ...simpleConfig.default, ...cliConfig };
         }
+
 		
 		
         const result = await main(finalConfig);        
@@ -35,9 +36,12 @@ import getCliParams from './lib/cli/cli.js';
 		const recordsPerSecond = result.eventCount / result.time.delta * 1000;
 		console.log(`⚡ Records per second: ${recordsPerSecond.toFixed(2)}`);
 		
+		// @ts-ignore
 		if (result.errors?.length) {
+			// @ts-ignore
 			console.error(`\n❗ Errors encountered: ${result.errors.length}`);
 			if (cliConfig.verbose) {
+				// @ts-ignore
 				result.errors.forEach(err => console.error(`   ${err}`));
 			}
 		} else {

package/index.js

@@ -29,16 +29,13 @@ import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 
 // Utilities
-import { generateLineChart } from './lib/utils/chart.js';
 
 // External dependencies
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import functions from '@google-cloud/functions-framework';
 import { timer, sLog } from 'ak-tools';
-import fs, { existsSync } from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
+import { existsSync } from 'fs';
 
 // Initialize dayjs and time constants
 dayjs.extend(utc);
@@ -66,7 +63,6 @@ function displayConfigurationSummary(config) {
 	if (config.hasAnonIds) features.push('anonymous IDs');
 	if (config.hasSessionIds) features.push('session IDs');
 	if (config.alsoInferFunnels) features.push('funnel inference');
-	if (config.makeChart) features.push('chart generation');
 	if (config.writeToDisk) features.push('disk output');
 	
 	if (features.length > 0) {
@@ -100,7 +96,10 @@ function displayConfigurationSummary(config) {
 	
 	// Group analytics
 	if (config.groupKeys && config.groupKeys.length > 0) {
-		const groups = config.groupKeys.map(([key, count]) => `${count} ${key}s`).join(', ');
+		const groups = config.groupKeys.map((group) => {
+			const [key, count] = Array.isArray(group) ? group : [group, 0];
+			return `${count} ${key}s`;
+		}).join(', ');
 		console.log(`👥 Groups:      ${groups}`);
 	}
 	
@@ -189,15 +188,12 @@ async function main(config) {
 
 		// ! DATA GENERATION ENDS HERE
 
-		// Step 10: Generate charts (if enabled)
-		if (validatedConfig.makeChart) {
-			await generateCharts(context);
+		// Step 10:  flush lookup tables to disk (always as CSVs)
+		if (validatedConfig.writeToDisk) {
+			await flushLookupTablesToDisk(storage, validatedConfig);
 		}
 
-		// Step 11a: Always flush lookup tables to disk (regardless of writeToDisk setting)
-		await flushLookupTablesToDisk(storage, validatedConfig);
-
-		// Step 11b: Flush other storage containers to disk (if writeToDisk enabled)
+		// Step 11: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
@@ -217,7 +213,7 @@ async function main(config) {
 		return {
 			...extractedData,
 			importResults,
-			files: extractFileInfo(storage),
+			files: await extractFileInfo(storage, validatedConfig),
 			time: { start, end, delta, human },
 			operations: context.getOperations(),
 			eventCount: context.getEventCount(),
@@ -416,28 +412,6 @@ async function generateGroupSCDs(context) {
 	}
 }
 
-/**
- * Generate charts for data visualization
- * @param {Context} context - Context object
- */
-async function generateCharts(context) {
-	const { config, storage } = context;
-
-	if (config.makeChart && storage.eventData?.length > 0) {
-		const chartPath = typeof config.makeChart === 'string'
-			? config.makeChart
-			: `./${config.simulationName}-timeline`;
-
-		await generateLineChart(storage.eventData, undefined, chartPath);
-
-		if (context.isCLI() || config.verbose) {
-			console.log(`📊 Chart generated: ${chartPath}`);
-		} else {
-			sLog("Chart generated", { path: chartPath });
-		}
-	}
-}
-
 /**
  * Flush lookup tables to disk (always runs, regardless of writeToDisk setting)
  * @param {import('./types').Storage} storage - Storage containers
@@ -502,11 +476,13 @@ async function flushStorageToDisk(storage, config) {
 /**
  * Extract file information from storage containers
  * @param {import('./types').Storage} storage - Storage object
- * @returns {string[]} Array of file paths
+ * @param {import('./types').Dungeon} config - Configuration object
+ * @returns {Promise<string[]>} Array of file paths
  */
-function extractFileInfo(storage) {
+async function extractFileInfo(storage, config) {
 	const files = [];
 
+	// Try to get paths from containers first
 	Object.values(storage).forEach(container => {
 		if (Array.isArray(container)) {
 			container.forEach(subContainer => {
@@ -519,6 +495,55 @@ function extractFileInfo(storage) {
 		}
 	});
 
+	// If no files found from containers and writeToDisk is enabled, scan the data directory
+	if (files.length === 0 && config.writeToDisk) {
+		try {
+			const fs = await import('fs');
+			const path = await import('path');
+			
+			let dataDir = path.resolve("./data");
+			if (!fs.existsSync(dataDir)) {
+				dataDir = path.resolve("./");
+			}
+			
+			if (fs.existsSync(dataDir)) {
+				const allFiles = fs.readdirSync(dataDir);
+				const simulationName = config.name;
+				
+				// Filter files that match our patterns and were likely created by this run
+				const relevantFiles = allFiles.filter(file => {
+					// Skip system files
+					if (file.startsWith('.')) return false;
+					
+					// If we have a simulation name, only include files with that prefix
+					if (simulationName && !file.startsWith(simulationName)) {
+						return false;
+					}
+					
+					// Check for common patterns
+					const hasEventPattern = file.includes('-EVENTS.');
+					const hasUserPattern = file.includes('-USERS.');
+					const hasScdPattern = file.includes('-SCD.');
+					const hasGroupPattern = file.includes('-GROUPS.');
+					const hasLookupPattern = file.includes('-LOOKUP.');
+					const hasAdspendPattern = file.includes('-ADSPEND.');
+					const hasMirrorPattern = file.includes('-MIRROR.');
+					
+					return hasEventPattern || hasUserPattern || hasScdPattern || 
+						   hasGroupPattern || hasLookupPattern || hasAdspendPattern || hasMirrorPattern;
+				});
+				
+				// Convert to full paths
+				relevantFiles.forEach(file => {
+					files.push(path.join(dataDir, file));
+				});
+			}
+		} catch (error) {
+			// If scanning fails, just return empty array
+			console.warn('Warning: Could not scan data directory for files:', error.message);
+		}
+	}
+
 	return files;
 }
 

package/lib/cli/cli.js

@@ -160,13 +160,6 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("makeChart", {
-			alias: 'mc',
-			demandOption: false,
-			describe: 'create a PNG chart from data',
-			type: 'boolean',
-			coerce: boolCoerce
-		})
 		.option("hasAdSpend", {
 			alias: 'ads',
 			demandOption: false,
@@ -223,6 +216,12 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.option("name", {
+			alias: 'n',
+			demandOption: false,
+			describe: 'custom name for generated files (prefix)',
+			type: 'string'
+		})
 
 		.help()
 		.wrap(null)
@@ -236,7 +235,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 }
 
 
-function boolCoerce(value, foo) {
+function boolCoerce(value) {
 	if (typeof value === 'boolean') return value;
 	if (typeof value === 'string') {
 		return value.toLowerCase() === 'true';

package/lib/core/config-validator.js

@@ -101,7 +101,6 @@ export function validateDungeonConfig(config) {
 		region = "US",
 		writeToDisk = false,
 		verbose = true,
-		makeChart = false,
 		soup = {},
 		hook = (record) => record,
 		hasAdSpend = false,
@@ -115,13 +114,13 @@ export function validateDungeonConfig(config) {
 		hasIOSDevices = false,
 		alsoInferFunnels = false,
 		name = "",
-		batchSize = 500_000,
-		concurrency
+		batchSize = 2_500_000,
+		concurrency = 1
 	} = config;
 
-	// Set concurrency default only if not provided
-	if (concurrency === undefined || concurrency === null) {
-		concurrency = Math.min(os.cpus().length * 2, 16);
+	// Allow concurrency override from config (default is now 1)
+	if (config.concurrency === undefined || config.concurrency === null) {
+		concurrency = 1;
 	}
 
 	// Ensure defaults for deep objects
@@ -136,9 +135,10 @@ export function validateDungeonConfig(config) {
 		throw new Error("Either epochStart or numDays must be provided");
 	}
 
-	// Generate simulation name
-	config.simulationName = name || makeName();
-	config.name = config.simulationName;
+	// Use provided name if non-empty string, otherwise generate one
+	if (!name || name === "") {
+		name = makeName();
+	}
 
 	// Validate events
 	if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
@@ -195,7 +195,7 @@ export function validateDungeonConfig(config) {
 	const definedEvents = events.map(e => e.event);
 	const missingEvents = eventInFunnels.filter(event => !definedEvents.includes(event));
 	if (missingEvents.length) {
-		throw new Error(`Funnel sequences contain events that are not defined in the events config:\n${missingEvents.join(', ')}\nPlease ensure all events in funnel sequences are defined in the events array.`);
+		throw new Error(`Funnel sequences contain events that are not defined in the events config:\n\n${missingEvents.join(', ')}\n\nPlease ensure all events in funnel sequences are defined in the events array.`);
 	}
 
 
@@ -230,7 +230,6 @@ export function validateDungeonConfig(config) {
 		region,
 		writeToDisk,
 		verbose,
-		makeChart,
 		soup,
 		hook,
 		hasAdSpend,
@@ -242,8 +241,7 @@ export function validateDungeonConfig(config) {
 		hasAndroidDevices,
 		hasDesktopDevices,
 		hasIOSDevices,
-		simulationName: config.simulationName,
-		name: config.name
+		name
 	};
 
 	return validatedConfig;

package/lib/core/context.js

@@ -40,6 +40,14 @@ function createDefaults(config, campaignData) {
 	const weighedBrowsers = u.weighArray(devices.browsers);
 	const weighedCampaigns = u.weighArray(campaignData);
 
+	// PERFORMANCE: Pre-compute device pools based on config to avoid rebuilding in makeEvent
+	const devicePools = {
+		android: config.hasAndroidDevices ? weighedAndroidDevices : [],
+		ios: config.hasIOSDevices ? weighedIOSDevices : [],
+		desktop: config.hasDesktopDevices ? weighedDesktopDevices : []
+	};
+	const allDevices = [...devicePools.android, ...devicePools.ios, ...devicePools.desktop];
+
 	return {
 		locationsUsers: () => weighedLocationsUsers,
 		locationsEvents: () => weighedLocationsEvents,
@@ -47,7 +55,11 @@ function createDefaults(config, campaignData) {
 		androidDevices: () => weighedAndroidDevices,
 		desktopDevices: () => weighedDesktopDevices,
 		browsers: () => weighedBrowsers,
-		campaigns: () => weighedCampaigns
+		campaigns: () => weighedCampaigns,
+		
+		// PERFORMANCE: Pre-computed device pools
+		devicePools,
+		allDevices
 	};
 }
 

package/lib/core/storage.js

@@ -65,13 +65,15 @@ export async function createHookArray(arr = [], opts) {
 	}
 
 	function getWritePath() {
+		const gzipSuffix = (config.gzip && !writeDir?.startsWith('gs://')) ? '.gz' : '';
+		
 		if (isBatchMode) {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
-			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`);
 		}
 		else {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
-			return path.join(writeDir, `${filepath}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}.${format}${gzipSuffix}`);
 		}
 	}
 
@@ -153,12 +155,19 @@ export async function createHookArray(arr = [], opts) {
 			console.log(`\n\twriting ${writePath}\n`);
 		}
 
+		const streamOptions = {
+			gzip: config.gzip || false
+		};
+
 		switch (format) {
 			case "csv":
-				writeResult = await u.streamCSV(writePath, data);
+				writeResult = await u.streamCSV(writePath, data, streamOptions);
 				break;
 			case "json":
-				writeResult = await u.streamJSON(writePath, data);
+				writeResult = await u.streamJSON(writePath, data, streamOptions);
+				break;
+			case "parquet":
+				writeResult = await u.streamParquet(writePath, data, streamOptions);
 				break;
 			default:
 				throw new Error(`format ${format} is not supported`);
@@ -219,12 +228,15 @@ export class StorageManager {
 	async initializeContainers() {
 		const { config } = this.context;
 
+		// Validate configuration for potential data loss scenarios
+		this.validateConfiguration(config);
+
 		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
 				type: "event",
-				filepath: `${config.simulationName || 'events'}-EVENTS`,
+				filepath: `${config.name}-EVENTS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -233,7 +245,7 @@ export class StorageManager {
 			userProfilesData: await createHookArray([], {
 				hook: config.hook,
 				type: "user",
-				filepath: `${config.simulationName || 'users'}-USERS`,
+				filepath: `${config.name}-USERS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -242,7 +254,7 @@ export class StorageManager {
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
 				type: "ad-spend",
-				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
+				filepath: `${config.name}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -255,7 +267,7 @@ export class StorageManager {
 			mirrorEventData: await createHookArray([], {
 				hook: config.hook,
 				type: "mirror",
-				filepath: `${config.simulationName || 'mirror'}-MIRROR`,
+				filepath: `${config.name}-MIRROR`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -268,7 +280,7 @@ export class StorageManager {
 				const scdArray = await createHookArray([], {
 					hook: config.hook,
 					type: "scd",
-					filepath: `${config.simulationName || 'scd'}-${scdKey}-SCD`,
+					filepath: `${config.name}-${scdKey}-SCD`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -284,7 +296,7 @@ export class StorageManager {
 				const groupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "group",
-					filepath: `${config.simulationName || 'groups'}-${groupKey}-GROUPS`,
+					filepath: `${config.name}-${groupKey}-GROUPS`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -300,7 +312,7 @@ export class StorageManager {
 				const lookupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "lookup",
-					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
+					filepath: `${config.name}-${lookupConfig.key}-LOOKUP`,
 					format: "csv", // Always force CSV for lookup tables
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -312,4 +324,24 @@ export class StorageManager {
 
 		return storage;
 	}
+
+	/**
+	 * Validates configuration to prevent data loss scenarios
+	 * @param {Object} config - Configuration object
+	 */
+	validateConfiguration(config) {
+		// Check for potential data loss scenario: writeToDisk=false with low batchSize
+		if (config.writeToDisk === false) {
+			const batchSize = config.batchSize || 1_000_000;
+			const numEvents = config.numEvents || 0;
+			
+			if (batchSize < numEvents) {
+				throw new Error(
+					`Configuration error: writeToDisk is explicitly set to false but batchSize (${batchSize}) is lower than numEvents (${numEvents}). ` +
+					`This would result in data loss as batched data would be discarded. ` +
+					`Either set writeToDisk to true, increase batchSize to be >= numEvents, or provide a Mixpanel token to send data directly.`
+				);
+			}
+		}
+	}
 }

package/lib/generators/adspend.js

@@ -78,7 +78,7 @@ function createAdSpendEvent(network, campaign, day, chance) {
     return {
         event: "$ad_spend",
         time: day,
-        source: 'dm4',
+        // source: 'dm4',
         utm_campaign: campaign,
         campaign_id: id,
         insert_id: uid,

package/lib/generators/events.js

@@ -66,13 +66,12 @@ export async function makeEvent(
     // Create base event template
     const eventTemplate = {
         event: chosenEvent.event,
-        source: "dm4",
+        // source: "dm4",
         time: "",
         insert_id: "",
     };
 
     let defaultProps = {};
-    let devicePool = [];
 
     // Add default properties based on configuration
     if (hasLocation) {
@@ -82,32 +81,30 @@ export async function makeEvent(
     if (hasBrowser) {
         defaultProps.browser = u.choose(defaults.browsers());
     }
-    
-    // Build device pool based on enabled device types
-    if (hasAndroidDevices) devicePool.push(defaults.androidDevices());
-    if (hasIOSDevices) devicePool.push(defaults.iOSDevices());
-    if (hasDesktopDevices) devicePool.push(defaults.desktopDevices());
 
     // Add campaigns with attribution likelihood
     if (hasCampaigns && chance.bool({ likelihood: 25 })) {
         defaultProps.campaigns = u.pickRandom(defaults.campaigns());
     }
     
-    // Select device from pool
-    const devices = devicePool.flat();
-    if (devices.length) {
-        defaultProps.device = u.pickRandom(devices);
+    // PERFORMANCE: Use pre-computed device pool instead of rebuilding every time
+    if (defaults.allDevices.length) {
+        defaultProps.device = u.pickRandom(defaults.allDevices);
     }
 
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            // Apply time shift to move to present day using precomputed value
-            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Direct numeric calculation instead of dayjs object creation
+            const shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         } else {
-            // Get time from TimeSoup and apply precomputed time shift
+            // Get time from TimeSoup (returns ISO string) and apply precomputed time shift
             const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
-            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Parse ISO directly to milliseconds, add shift, convert back to ISO with one dayjs call
+            const soupTimestamp = new Date(soupTime).getTime() / 1000; // Convert to unix seconds
+            const shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         }
     }
 
@@ -133,7 +130,9 @@ export async function makeEvent(
     // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
     if (chosenEvent.properties) {
-        for (const key in chosenEvent.properties) {
+        const eventKeys = Object.keys(chosenEvent.properties);
+        for (let i = 0; i < eventKeys.length; i++) {
+            const key = eventKeys[i];
             try {
                 eventTemplate[key] = u.choose(chosenEvent.properties[key]);
             } catch (e) {
@@ -145,7 +144,9 @@ export async function makeEvent(
     
     // Add super properties (override event properties if needed)
     if (superProps) {
-        for (const key in superProps) {
+        const superKeys = Object.keys(superProps);
+        for (let i = 0; i < superKeys.length; i++) {
+            const key = superKeys[i];
             try {
                 eventTemplate[key] = u.choose(superProps[key]);
             } catch (e) {