make-mp-data 2.0.19 → 2.0.22

package/index.js

@@ -47,6 +47,65 @@ global.FIXED_NOW = FIXED_NOW;
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
 global.FIXED_BEGIN = FIXED_BEGIN;
 
+/**
+ * Display configuration summary for CLI mode
+ * @param {Config} config - Validated configuration object
+ */
+function displayConfigurationSummary(config) {
+	console.log('\n📋 Configuration Summary');
+	console.log('─'.repeat(40));
+	
+	// Core parameters
+	console.log(`🎯 Target:      ${config.numUsers?.toLocaleString()} users, ${config.numEvents?.toLocaleString()} events`);
+	console.log(`📅 Timeline:    ${config.numDays} days (${config.seed ? config.seed : 'random seed'})`);
+	console.log(`💾 Output:      ${config.format} format${config.region ? ` (${config.region})` : ''}`);
+	console.log(`⚡ Performance: ${config.concurrency || 'auto'} threads`);
+	
+	// Feature flags
+	const features = [];
+	if (config.hasAnonIds) features.push('anonymous IDs');
+	if (config.hasSessionIds) features.push('session IDs');
+	if (config.alsoInferFunnels) features.push('funnel inference');
+	if (config.makeChart) features.push('chart generation');
+	if (config.writeToDisk) features.push('disk output');
+	
+	if (features.length > 0) {
+		console.log(`🔧 Features:    ${features.join(', ')}`);
+	}
+	
+	// Schema preview
+	if (config.events && config.events.length > 0) {
+		console.log('\n🎭 Event Schema');
+		console.log('─'.repeat(40));
+		const eventNames = config.events.slice(0, 6).map(e => e.event || e).join(', ');
+		const more = config.events.length > 6 ? ` (+${config.events.length - 6} more)` : '';
+		console.log(`📊 Events:      ${eventNames}${more}`);
+	}
+	
+	// Funnels preview
+	if (config.funnels && config.funnels.length > 0) {
+		console.log(`🔄 Funnels:     ${config.funnels.length} funnel${config.funnels.length > 1 ? 's' : ''} configured`);
+		config.funnels.slice(0, 4).forEach((funnel, i) => {
+			if (funnel.sequence) {
+				const arrow = ' → ';
+				const sequence = funnel.sequence.join(arrow);
+				const rate = funnel.conversionRate ? ` (${(funnel.conversionRate).toFixed(0)}% conversion)` : '';
+				console.log(`    ${i + 1}. ${sequence}${rate}`);
+			}
+		});
+		if (config.funnels.length > 4) {
+			console.log(`    ...and ${config.funnels.length - 4} more funnels`);
+		}
+	}
+	
+	// Group analytics
+	if (config.groupKeys && config.groupKeys.length > 0) {
+		const groups = config.groupKeys.map(([key, count]) => `${count} ${key}s`).join(', ');
+		console.log(`👥 Groups:      ${groups}`);
+	}
+	
+	console.log(''); // Extra spacing before generation starts
+}
 
 /**
  * Main data generation function
@@ -81,6 +140,11 @@ async function main(config) {
 	try {
 		// Step 1: Validate and enrich configuration
 		validatedConfig = validateDungeonConfig(config);
+		
+		// Step 1.5: Display configuration summary (CLI mode only)
+		if (isCLI && validatedConfig.verbose) {
+			displayConfigurationSummary(validatedConfig);
+		}
 
 		// Step 2: Create context with validated config
 		const context = createContext(validatedConfig, null, isCLI);
@@ -90,6 +154,8 @@ async function main(config) {
 		const storage = await storageManager.initializeContainers();
 		updateContextWithStorage(context, storage);
 
+		// ! DATA GENERATION STARTS HERE
+
 		// Step 4: Generate ad spend data (if enabled)
 		if (validatedConfig.hasAdSpend) {
 			await generateAdSpendData(context);
@@ -119,12 +185,19 @@ async function main(config) {
 			await makeMirror(context);
 		}
 
+		if (context.config.verbose) console.log(`\n✅ Data generation completed successfully!\n`);
+
+		// ! DATA GENERATION ENDS HERE
+
 		// Step 10: Generate charts (if enabled)
 		if (validatedConfig.makeChart) {
 			await generateCharts(context);
 		}
 
-		// Step 11: Flush storage containers to disk (if writeToDisk enabled)
+		// Step 11a: Always flush lookup tables to disk (regardless of writeToDisk setting)
+		await flushLookupTablesToDisk(storage, validatedConfig);
+
+		// Step 11b: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
@@ -252,7 +325,8 @@ async function generateLookupTables(context) {
 
 		for (let j = 0; j < entries; j++) {
 			const lookupEntry = await makeProfile(context, attributes, {
-				[key]: `${key}_${j + 1}`
+				id: j + 1 //primary key is always a number so it joins simply with events
+				// [key]: `${key}_${j + 1}` // we don't want to use the lookup name as a prefix here
 			});
 
 			await lookupContainer.hookPush(lookupEntry);
@@ -352,7 +426,7 @@ async function generateCharts(context) {
 	if (config.makeChart && storage.eventData?.length > 0) {
 		const chartPath = typeof config.makeChart === 'string'
 			? config.makeChart
-			: `./charts/${config.simulationName}-timeline.png`;
+			: `./${config.simulationName}-timeline`;
 
 		await generateLineChart(storage.eventData, undefined, chartPath);
 
@@ -365,7 +439,33 @@ async function generateCharts(context) {
 }
 
 /**
- * Flush all storage containers to disk
+ * Flush lookup tables to disk (always runs, regardless of writeToDisk setting)
+ * @param {import('./types').Storage} storage - Storage containers
+ * @param {import('./types').Dungeon} config - Configuration object
+ */
+async function flushLookupTablesToDisk(storage, config) {
+	if (!storage.lookupTableData || !Array.isArray(storage.lookupTableData) || storage.lookupTableData.length === 0) {
+		return; // No lookup tables to flush
+	}
+
+	if (config.verbose) {
+		console.log('💾 Writing lookup tables to disk...');
+	}
+
+	const flushPromises = [];
+	storage.lookupTableData.forEach(container => {
+		if (container?.flush) flushPromises.push(container.flush());
+	});
+
+	await Promise.all(flushPromises);
+
+	if (config.verbose) {
+		console.log('🗂️  Lookup tables flushed to disk successfully');
+	}
+}
+
+/**
+ * Flush all storage containers to disk (excluding lookup tables)
  * @param {import('./types').Storage} storage - Storage containers
  * @param {import('./types').Dungeon} config - Configuration object
  */
@@ -383,8 +483,8 @@ async function flushStorageToDisk(storage, config) {
 	if (storage.mirrorEventData?.flush) flushPromises.push(storage.mirrorEventData.flush());
 	if (storage.groupEventData?.flush) flushPromises.push(storage.groupEventData.flush());
 
-	// Flush arrays of HookedArrays
-	[storage.scdTableData, storage.groupProfilesData, storage.lookupTableData].forEach(arrayOfContainers => {
+	// Flush arrays of HookedArrays (excluding lookup tables which are handled separately)
+	[storage.scdTableData, storage.groupProfilesData].forEach(arrayOfContainers => {
 		if (Array.isArray(arrayOfContainers)) {
 			arrayOfContainers.forEach(container => {
 				if (container?.flush) flushPromises.push(container.flush());
@@ -395,7 +495,7 @@ async function flushStorageToDisk(storage, config) {
 	await Promise.all(flushPromises);
 
 	if (config.verbose) {
-		console.log('✅ Data flushed to disk successfully');
+		console.log('🙏 Data flushed to disk successfully');
 	}
 }
 

package/lib/cli/cli.js

@@ -121,6 +121,14 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.options("sanity", {
+			demandOption: false,
+			default: false,
+			describe: 'run sanity checks on the generated data',
+			alias: 'san',
+			type: 'boolean',
+			coerce: boolCoerce
+		})
 		.option("writeToDisk", {
 			demandOption: false,
 			default: true,