make-mp-data 2.0.23 → 2.1.1

package/dungeons/userAgent.js

@@ -14,7 +14,7 @@ const days = 30;
 
 /** @type {Config} */
 const config = {
-	token: "a854e82c2c91ada31c502139d99e8e0d",
+	token: "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 100,
@@ -33,7 +33,6 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
-	makeChart: false,
 
 	batchSize: 500_000,
 	concurrency: 500,

package/entry.js

@@ -36,9 +36,12 @@ import getCliParams from './lib/cli/cli.js';
 		const recordsPerSecond = result.eventCount / result.time.delta * 1000;
 		console.log(`⚡ Records per second: ${recordsPerSecond.toFixed(2)}`);
 		
+		// @ts-ignore
 		if (result.errors?.length) {
+			// @ts-ignore
 			console.error(`\n❗ Errors encountered: ${result.errors.length}`);
 			if (cliConfig.verbose) {
+				// @ts-ignore
 				result.errors.forEach(err => console.error(`   ${err}`));
 			}
 		} else {

package/index.js

@@ -29,16 +29,13 @@ import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 
 // Utilities
-import { generateLineChart } from './lib/utils/chart.js';
 
 // External dependencies
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import functions from '@google-cloud/functions-framework';
 import { timer, sLog } from 'ak-tools';
-import fs, { existsSync } from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
+import { existsSync } from 'fs';
 
 // Initialize dayjs and time constants
 dayjs.extend(utc);
@@ -66,7 +63,6 @@ function displayConfigurationSummary(config) {
 	if (config.hasAnonIds) features.push('anonymous IDs');
 	if (config.hasSessionIds) features.push('session IDs');
 	if (config.alsoInferFunnels) features.push('funnel inference');
-	if (config.makeChart) features.push('chart generation');
 	if (config.writeToDisk) features.push('disk output');
 	
 	if (features.length > 0) {
@@ -100,7 +96,10 @@ function displayConfigurationSummary(config) {
 	
 	// Group analytics
 	if (config.groupKeys && config.groupKeys.length > 0) {
-		const groups = config.groupKeys.map(([key, count]) => `${count} ${key}s`).join(', ');
+		const groups = config.groupKeys.map((group) => {
+			const [key, count] = Array.isArray(group) ? group : [group, 0];
+			return `${count} ${key}s`;
+		}).join(', ');
 		console.log(`👥 Groups:      ${groups}`);
 	}
 	
@@ -189,27 +188,23 @@ async function main(config) {
 
 		// ! DATA GENERATION ENDS HERE
 
-		// Step 10: Generate charts (if enabled)
-		if (validatedConfig.makeChart) {
-			await generateCharts(context);
+		// Step 10: Send to Mixpanel (if token provided)
+		// IMPORTANT: Must happen BEFORE flushing to disk, because flush() clears the arrays
+		let importResults;
+		if (validatedConfig.token) {
+			importResults = await sendToMixpanel(context);
 		}
 
-		// Step 11a:  flush lookup tables to disk (always as CSVs)
+		// Step 11: Flush lookup tables to disk (always as CSVs)
 		if (validatedConfig.writeToDisk) {
 			await flushLookupTablesToDisk(storage, validatedConfig);
 		}
 
-		// Step 11b: Flush other storage containers to disk (if writeToDisk enabled)
+		// Step 12: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
 
-		// Step 12: Send to Mixpanel (if token provided)
-		let importResults;
-		if (validatedConfig.token) {
-			importResults = await sendToMixpanel(context);
-		}
-
 		// Step 13: Compile results
 		jobTimer.stop(false);
 		const { start, end, delta, human } = jobTimer.report(false);
@@ -418,28 +413,6 @@ async function generateGroupSCDs(context) {
 	}
 }
 
-/**
- * Generate charts for data visualization
- * @param {Context} context - Context object
- */
-async function generateCharts(context) {
-	const { config, storage } = context;
-
-	if (config.makeChart && storage.eventData?.length > 0) {
-		const chartPath = typeof config.makeChart === 'string'
-			? config.makeChart
-			: `./${config.name}-timeline`;
-
-		await generateLineChart(storage.eventData, undefined, chartPath);
-
-		if (context.isCLI() || config.verbose) {
-			console.log(`📊 Chart generated: ${chartPath}`);
-		} else {
-			sLog("Chart generated", { path: chartPath });
-		}
-	}
-}
-
 /**
  * Flush lookup tables to disk (always runs, regardless of writeToDisk setting)
  * @param {import('./types').Storage} storage - Storage containers
@@ -505,7 +478,7 @@ async function flushStorageToDisk(storage, config) {
  * Extract file information from storage containers
  * @param {import('./types').Storage} storage - Storage object
  * @param {import('./types').Dungeon} config - Configuration object
- * @returns {string[]} Array of file paths
+ * @returns {Promise<string[]>} Array of file paths
  */
 async function extractFileInfo(storage, config) {
 	const files = [];

package/lib/cli/cli.js

@@ -160,13 +160,6 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("makeChart", {
-			alias: 'mc',
-			demandOption: false,
-			describe: 'create a PNG chart from data',
-			type: 'boolean',
-			coerce: boolCoerce
-		})
 		.option("hasAdSpend", {
 			alias: 'ads',
 			demandOption: false,

package/lib/core/config-validator.js

@@ -101,7 +101,6 @@ export function validateDungeonConfig(config) {
 		region = "US",
 		writeToDisk = false,
 		verbose = true,
-		makeChart = false,
 		soup = {},
 		hook = (record) => record,
 		hasAdSpend = false,
@@ -115,13 +114,13 @@ export function validateDungeonConfig(config) {
 		hasIOSDevices = false,
 		alsoInferFunnels = false,
 		name = "",
-		batchSize = 500_000,
-		concurrency
+		batchSize = 2_500_000,
+		concurrency = 1
 	} = config;
 
-	// Set concurrency default only if not provided
-	if (concurrency === undefined || concurrency === null) {
-		concurrency = Math.min(os.cpus().length * 2, 16);
+	// Allow concurrency override from config (default is now 1)
+	if (config.concurrency === undefined || config.concurrency === null) {
+		concurrency = 1;
 	}
 
 	// Ensure defaults for deep objects
@@ -196,7 +195,7 @@ export function validateDungeonConfig(config) {
 	const definedEvents = events.map(e => e.event);
 	const missingEvents = eventInFunnels.filter(event => !definedEvents.includes(event));
 	if (missingEvents.length) {
-		throw new Error(`Funnel sequences contain events that are not defined in the events config:\n${missingEvents.join(', ')}\nPlease ensure all events in funnel sequences are defined in the events array.`);
+		throw new Error(`Funnel sequences contain events that are not defined in the events config:\n\n${missingEvents.join(', ')}\n\nPlease ensure all events in funnel sequences are defined in the events array.`);
 	}
 
 
@@ -231,7 +230,6 @@ export function validateDungeonConfig(config) {
 		region,
 		writeToDisk,
 		verbose,
-		makeChart,
 		soup,
 		hook,
 		hasAdSpend,

package/lib/generators/adspend.js

@@ -78,7 +78,7 @@ function createAdSpendEvent(network, campaign, day, chance) {
     return {
         event: "$ad_spend",
         time: day,
-        source: 'dm4',
+        // source: 'dm4',
         utm_campaign: campaign,
         campaign_id: id,
         insert_id: uid,

package/lib/generators/events.js

@@ -66,7 +66,7 @@ export async function makeEvent(
     // Create base event template
     const eventTemplate = {
         event: chosenEvent.event,
-        source: "dm4",
+        // source: "dm4",
         time: "",
         insert_id: "",
     };