make-mp-data 2.0.0 → 2.0.2

package/index.js

@@ -29,8 +29,6 @@ import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 
 // Utilities
-import getCliParams from './lib/cli/cli.js';
-import * as u from './lib/utils/utils.js';
 import { generateLineChart } from './lib/utils/chart.js';
 
 // External dependencies
@@ -39,6 +37,8 @@ import utc from "dayjs/plugin/utc.js";
 import functions from '@google-cloud/functions-framework';
 import { timer, sLog } from 'ak-tools';
 import fs, { existsSync } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
 
 // Initialize dayjs and time constants
 dayjs.extend(utc);
@@ -47,12 +47,65 @@ global.FIXED_NOW = FIXED_NOW;
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
 global.FIXED_BEGIN = FIXED_BEGIN;
 
-// Package version
-const { version } = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
-
-// Environment
-const { NODE_ENV = "unknown" } = process.env;
-const isCLI = process.argv[1].endsWith('index.js') || process.argv[1].endsWith('cli.js');
+/**
+ * Display configuration summary for CLI mode
+ * @param {Config} config - Validated configuration object
+ */
+function displayConfigurationSummary(config) {
+	console.log('\n📋 Configuration Summary');
+	console.log('─'.repeat(40));
+	
+	// Core parameters
+	console.log(`🎯 Target:      ${config.numUsers?.toLocaleString()} users, ${config.numEvents?.toLocaleString()} events`);
+	console.log(`📅 Timeline:    ${config.numDays} days (${config.seed ? config.seed : 'random seed'})`);
+	console.log(`💾 Output:      ${config.format} format${config.region ? ` (${config.region})` : ''}`);
+	console.log(`⚡ Performance: ${config.concurrency || 'auto'} threads`);
+	
+	// Feature flags
+	const features = [];
+	if (config.hasAnonIds) features.push('anonymous IDs');
+	if (config.hasSessionIds) features.push('session IDs');
+	if (config.alsoInferFunnels) features.push('funnel inference');
+	if (config.makeChart) features.push('chart generation');
+	if (config.writeToDisk) features.push('disk output');
+	
+	if (features.length > 0) {
+		console.log(`🔧 Features:    ${features.join(', ')}`);
+	}
+	
+	// Schema preview
+	if (config.events && config.events.length > 0) {
+		console.log('\n🎭 Event Schema');
+		console.log('─'.repeat(40));
+		const eventNames = config.events.slice(0, 6).map(e => e.event || e).join(', ');
+		const more = config.events.length > 6 ? ` (+${config.events.length - 6} more)` : '';
+		console.log(`📊 Events:      ${eventNames}${more}`);
+	}
+	
+	// Funnels preview
+	if (config.funnels && config.funnels.length > 0) {
+		console.log(`🔄 Funnels:     ${config.funnels.length} funnel${config.funnels.length > 1 ? 's' : ''} configured`);
+		config.funnels.slice(0, 4).forEach((funnel, i) => {
+			if (funnel.sequence) {
+				const arrow = ' → ';
+				const sequence = funnel.sequence.join(arrow);
+				const rate = funnel.conversionRate ? ` (${(funnel.conversionRate * 100).toFixed(0)}% conversion)` : '';
+				console.log(`    ${i + 1}. ${sequence}${rate}`);
+			}
+		});
+		if (config.funnels.length > 4) {
+			console.log(`    ...and ${config.funnels.length - 4} more funnels`);
+		}
+	}
+	
+	// Group analytics
+	if (config.groupKeys && config.groupKeys.length > 0) {
+		const groups = config.groupKeys.map(([key, count]) => `${count} ${key}s`).join(', ');
+		console.log(`👥 Groups:      ${groups}`);
+	}
+	
+	console.log(''); // Extra spacing before generation starts
+}
 
 /**
  * Main data generation function
@@ -64,8 +117,9 @@ async function main(config) {
 	jobTimer.start();
 
 	//cli mode check for positional dungeon config
+	const isCLI = config._ && Array.isArray(config._);
 	if (isCLI) {
-		const firstArg = config._.slice().pop()
+		const firstArg = config._.slice().pop();
 		if (firstArg?.endsWith('.js') && existsSync(firstArg)) {
 			if (config.verbose) {
 				console.log(`\n🔍 Loading dungeon config from: ${firstArg}`);
@@ -78,27 +132,36 @@ async function main(config) {
 				throw error;
 			}
 		}
-		
+
 	}
 
+	if (config.verbose) console.log(`\n🔧 Configuring dungeon with seed: ${config.seed}`);
 	let validatedConfig;
 	try {
 		// Step 1: Validate and enrich configuration
 		validatedConfig = validateDungeonConfig(config);
+		
+		// Step 1.5: Display configuration summary (CLI mode only)
+		if (isCLI && validatedConfig.verbose) {
+			displayConfigurationSummary(validatedConfig);
+		}
 
 		// Step 2: Create context with validated config
-		const context = createContext(validatedConfig);
+		const context = createContext(validatedConfig, null, isCLI);
 
 		// Step 3: Initialize storage containers
 		const storageManager = new StorageManager(context);
 		const storage = await storageManager.initializeContainers();
 		updateContextWithStorage(context, storage);
 
+		// ! DATA GENERATION STARTS HERE
+
 		// Step 4: Generate ad spend data (if enabled)
 		if (validatedConfig.hasAdSpend) {
 			await generateAdSpendData(context);
 		}
 
+		if (context.config.verbose) console.log(`\n🔄 Starting user and event generation...\n`);
 		// Step 5: Main user and event generation
 		await userLoop(context);
 
@@ -122,6 +185,10 @@ async function main(config) {
 			await makeMirror(context);
 		}
 
+		if (context.config.verbose) console.log(`\n✅ Data generation completed successfully!\n`);
+
+		// ! DATA GENERATION ENDS HERE
+
 		// Step 10: Generate charts (if enabled)
 		if (validatedConfig.makeChart) {
 			await generateCharts(context);
@@ -155,11 +222,9 @@ async function main(config) {
 		};
 
 	} catch (error) {
-		if (isCLI || validatedConfig.verbose) {
+		if (validatedConfig?.verbose) {
 			console.error(`\n❌ Error: ${error.message}\n`);
-			if (validatedConfig.verbose) {
-				console.error(error.stack);
-			}
+			console.error(error.stack);
 		} else {
 			sLog("Main execution error", { error: error.message, stack: error.stack }, "ERROR");
 		}
@@ -195,7 +260,7 @@ async function generateGroupProfiles(context) {
 	const { config, storage } = context;
 	const { groupKeys, groupProps = {} } = config;
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('\n👥 Generating group profiles...');
 	}
 
@@ -208,7 +273,7 @@ async function generateGroupProfiles(context) {
 			continue;
 		}
 
-		if (isCLI || config.verbose) {
+		if (context.isCLI() || config.verbose) {
 			console.log(`   Creating ${groupCount.toLocaleString()} ${groupKey} profiles...`);
 		}
 
@@ -224,7 +289,7 @@ async function generateGroupProfiles(context) {
 		}
 	}
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('✅ Group profiles generated successfully');
 	}
 }
@@ -237,7 +302,7 @@ async function generateLookupTables(context) {
 	const { config, storage } = context;
 	const { lookupTables } = config;
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('\n🔍 Generating lookup tables...');
 	}
 
@@ -251,7 +316,7 @@ async function generateLookupTables(context) {
 			continue;
 		}
 
-		if (isCLI || config.verbose) {
+		if (context.isCLI() || config.verbose) {
 			console.log(`   Creating ${entries.toLocaleString()} ${key} lookup entries...`);
 		}
 
@@ -264,7 +329,7 @@ async function generateLookupTables(context) {
 		}
 	}
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('✅ Lookup tables generated successfully');
 	}
 }
@@ -277,7 +342,7 @@ async function generateGroupSCDs(context) {
 	const { config, storage } = context;
 	const { scdProps, groupKeys } = config;
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('\n📊 Generating group SCDs...');
 	}
 
@@ -300,7 +365,7 @@ async function generateGroupSCDs(context) {
 			continue; // No SCDs for this group type
 		}
 
-		if (isCLI || config.verbose) {
+		if (context.isCLI() || config.verbose) {
 			console.log(`   Generating SCDs for ${groupCount.toLocaleString()} ${groupKey} entities...`);
 		}
 
@@ -342,7 +407,7 @@ async function generateGroupSCDs(context) {
 		}
 	}
 
-	if (isCLI || config.verbose) {
+	if (context.isCLI() || config.verbose) {
 		console.log('✅ Group SCDs generated successfully');
 	}
 }
@@ -357,11 +422,11 @@ async function generateCharts(context) {
 	if (config.makeChart && storage.eventData?.length > 0) {
 		const chartPath = typeof config.makeChart === 'string'
 			? config.makeChart
-			: `./charts/${config.simulationName}-timeline.png`;
+			: `./${config.simulationName}-timeline`;
 
 		await generateLineChart(storage.eventData, undefined, chartPath);
 
-		if (isCLI || config.verbose) {
+		if (context.isCLI() || config.verbose) {
 			console.log(`📊 Chart generated: ${chartPath}`);
 		} else {
 			sLog("Chart generated", { path: chartPath });
@@ -375,7 +440,7 @@ async function generateCharts(context) {
  * @param {import('./types').Dungeon} config - Configuration object
  */
 async function flushStorageToDisk(storage, config) {
-	if (isCLI || config.verbose) {
+	if (config.verbose) {
 		console.log('\n💾 Writing data to disk...');
 	}
 
@@ -399,8 +464,8 @@ async function flushStorageToDisk(storage, config) {
 
 	await Promise.all(flushPromises);
 
-	if (isCLI || config.verbose) {
-		console.log('✅ Data flushed to disk successfully');
+	if (config.verbose) {
+		console.log('🙏 Data flushed to disk successfully');
 	}
 }
 
@@ -445,44 +510,6 @@ function extractStorageData(storage) {
 	};
 }
 
-// CLI execution
-if (isCLI) {
-	(async () => {
-		const cliConfig = getCliParams();
-
-		// Load dungeon config if --complex or --simple flags are used
-		let finalConfig = cliConfig;
-		if (cliConfig.complex) {
-			const complexConfig = await import('./dungeons/complex.js');
-			finalConfig = { ...complexConfig.default, ...cliConfig };
-		} else if (cliConfig.simple) {
-			const simpleConfig = await import('./dungeons/simple.js');
-			finalConfig = { ...simpleConfig.default, ...cliConfig };
-		}
-
-		main(finalConfig)
-			.then(result => {
-				console.log(`📊 Generated ${(result.eventCount || 0).toLocaleString()} events for ${(result.userCount || 0).toLocaleString()} users`);
-				console.log(`⏱️  Total time: ${result.time?.human || 'unknown'}`);
-				if (result.files?.length) {
-					console.log(`📁 Files written: ${result.files.length}`);
-					if (cliConfig.verbose) {
-						result.files.forEach(file => console.log(`   ${file}`));
-					}
-				}
-				console.log(`\n✅ Job completed successfully!`);
-				process.exit(0);
-			})
-			.catch(error => {
-				console.error(`\n❌ Job failed: ${error.message}`);
-				if (cliConfig.verbose) {
-					console.error(error.stack);
-				}
-				process.exit(1);
-			});
-	})();
-}
-
 // Cloud Functions setup
 functions.http('entry', async (req, res) => {
 	await handleCloudFunctionEntry(req, res, main);
@@ -494,4 +521,5 @@ export default main;
 // CommonJS compatibility
 if (typeof module !== 'undefined' && module.exports) {
 	module.exports = main;
-}
+}
+

package/lib/cli/cli.js

@@ -1,9 +1,13 @@
 import yargs from 'yargs';
 import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
 
 /** @typedef {import('../../types').Dungeon} Config */
 
-const { version } = JSON.parse(fs.readFileSync('package.json', 'utf8'));
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const packageJsonPath = path.join(__dirname, '../../package.json');
+const { version } = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
 
 const hero = String.raw`
                         
@@ -24,9 +28,10 @@ function cliParams() {
 		.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
 
 		examples:
-npx $0
-npx $0 --token 1234 --u 100 --e 1000 --d 7 --w false
-npx $0 myDataConfig.js
+npx $0                                    (uses simple data model by default)
+npx $0 --complex                          (full enterprise data model)
+npx $0 --token 1234 --u 100 --e 1000     (send data to mixpanel)
+npx $0 myDataConfig.js                    (use custom config)
 
 DOCS: https://github.com/ak--47/make-mp-data
 DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
@@ -95,7 +100,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option('concurrency', {
 			alias: 'conn',
-			default: 500,
+			default: 10,
 			demandOption: false,
 			describe: 'concurrency level for data generation',
 			type: 'number'

package/lib/core/config-validator.js

@@ -3,14 +3,20 @@
  * Extracted from index.js validateDungeonConfig function
  */
 
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+/** @typedef {import('../../types.js').EventConfig} EventConfig */
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').Funnel} Funnel */
+
 import dayjs from "dayjs";
-import { makeName, clone } from "ak-tools";
+import { makeName } from "ak-tools";
 import * as u from "../utils/utils.js";
+import os from "os";
 
 /**
  * Infers funnels from the provided events
- * @param {Array} events - Array of event configurations
- * @returns {Array} Array of inferred funnel configurations
+ * @param {EventConfig[]} events - Array of event configurations
+ * @returns {Funnel[]} Array of inferred funnel configurations
  */
 function inferFunnels(events) {
     const createdFunnels = [];
@@ -34,7 +40,7 @@ function inferFunnels(events) {
     if (firstEvents.length) {
         for (const event of firstEvents) {
             createdFunnels.push({ 
-                ...clone(funnelTemplate), 
+                ...u.deepClone(funnelTemplate), 
                 sequence: [event], 
                 isFirstFunnel: true, 
                 conversionRate: 100 
@@ -43,12 +49,12 @@ function inferFunnels(events) {
     }
 
     // At least one funnel with all usage events
-    createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+    createdFunnels.push({ ...u.deepClone(funnelTemplate), sequence: usageEvents });
 
     // Create random funnels for the rest
     for (let i = 1; i < numFunnelsToCreate; i++) {
         /** @type {import('../../types.js').Funnel} */
-        const funnel = { ...clone(funnelTemplate) };
+        const funnel = { ...u.deepClone(funnelTemplate) };
         funnel.conversionRate = u.integer(25, 75);
         funnel.timeToConvert = u.integer(1, 10);
         funnel.weight = u.integer(1, 10);
@@ -63,8 +69,8 @@ function inferFunnels(events) {
 
 /**
  * Validates and enriches a dungeon configuration object
- * @param {Object} config - Raw configuration object
- * @returns {Object} Validated and enriched configuration
+ * @param {Partial<Dungeon>} config - Raw configuration object
+ * @returns {Dungeon} Validated and enriched configuration
  */
 export function validateDungeonConfig(config) {
     const chance = u.getChance();
@@ -94,7 +100,7 @@ export function validateDungeonConfig(config) {
         token = null,
         region = "US",
         writeToDisk = false,
-        verbose = false,
+        verbose = true,
         makeChart = false,
         soup = {},
         hook = (record) => record,
@@ -110,9 +116,14 @@ export function validateDungeonConfig(config) {
         alsoInferFunnels = false,
         name = "",
         batchSize = 500_000,
-        concurrency = 500
+        concurrency
     } = config;
 
+    // Set concurrency default only if not provided
+    if (concurrency === undefined || concurrency === null) {
+        concurrency = Math.min(os.cpus().length * 2, 16);
+    }
+
     // Ensure defaults for deep objects
     if (!config.superProps) config.superProps = superProps;
     if (!config.userProps || Object.keys(config?.userProps || {})) config.userProps = userProps;
@@ -132,9 +143,9 @@ export function validateDungeonConfig(config) {
     // Validate events
     if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
     
-    // Convert string events to objects
+    // Convert string events to objects  
     if (typeof events[0] === "string") {
-        events = events.map(e => ({ event: e }));
+        events = events.map(e => ({ event: /** @type {string} */ (e) }));
     }
 
     // Handle funnel inference
@@ -232,6 +243,11 @@ export function validateDungeonConfig(config) {
  * @param {Object} config - Configuration to validate
  * @throws {Error} If required fields are missing
  */
+/**
+ * Validates required configuration parameters
+ * @param {Dungeon} config - Configuration object to validate
+ * @returns {boolean} True if validation passes
+ */
 export function validateRequiredConfig(config) {
     if (!config) {
         throw new Error("Configuration is required");