make-mp-data 2.1.6 → 3.0.1

package/dungeons/too-big-events.js

@@ -0,0 +1,166 @@
+/**
+ * "Too Big Events" dungeon
+ * Tests Mixpanel row limit (1MB) and column limit (8KB for arrays of objects)
+ * Generates events with absurd numbers of properties and oversized array-of-object columns
+ */
+
+function integer(min = 1, max = 100) {
+	if (min === max) return min;
+	if (min > max) [min, max] = [max, min];
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function loremChunk(size = 200) {
+	const words = ["foo", "bar", "baz", "qux", "garply", "waldo", "fred", "plugh", "xyzzy", "thud", "corge", "grault", "flob", "zorp", "narf", "blip", "snork", "quux", "wobble", "crumn"];
+	let result = "";
+	while (result.length < size) {
+		result += words[Math.floor(Math.random() * words.length)] + " ";
+	}
+	return result.trim();
+}
+
+// Generate a fat array of objects (targeting >8KB per column)
+function bigArrayOfObjects(numItems = 50) {
+	numItems = numItems * 2
+	return () => {
+		const arr = [];
+		for (let i = 0; i < numItems; i++) {
+			arr.push({
+				id: `item_${i}_${Math.random().toString(36).slice(2, 10)}`,
+				name: loremChunk(80),
+				description: loremChunk(150),
+				category: ["foo", "bar", "baz", "qux"][Math.floor(Math.random() * 4)],
+				price: parseFloat((Math.random() * 999).toFixed(2)),
+				quantity: integer(1, 100),
+				tags: ["alpha", "beta", "gamma", "delta", "epsilon"].slice(0, integer(1, 5)),
+				metadata: {
+					source: loremChunk(40),
+					ref: Math.random().toString(36).slice(2, 14),
+					ts: new Date().toISOString()
+				}
+			});
+		}
+		return arr;
+	};
+}
+
+// Generate a truly massive array of objects (way over 8KB)
+function hugeArrayOfObjects() {
+	return bigArrayOfObjects(150)();
+}
+
+// Build a ton of flat properties to bloat row size toward 1MB
+function manyProperties(count = 200) {
+	count = count * 2
+	const props = {};
+	for (let i = 0; i < count; i++) {
+		const type = i % 4;
+		if (type === 0) props[`str_prop_${i}`] = () => loremChunk(300);
+		else if (type === 1) props[`num_prop_${i}`] = () => integer(1, 999999);
+		else if (type === 2) props[`bool_prop_${i}`] = [true, false];
+		else props[`list_prop_${i}`] = () => Array.from({ length: integer(5, 20) }, () => loremChunk(50));
+	}
+	return props;
+}
+
+const seed = "too-big-" + Math.random().toString(36).slice(2, 8);
+
+/** @type {import('../types').Dungeon} */
+const config = {
+	token: "",
+	seed: seed,
+	numDays: 30,
+	numEvents: 1000,
+	numUsers: 50,
+	format: 'json',
+	region: "US",
+	hasAnonIds: false,
+	hasSessionIds: false,
+	batchSize: 10000,
+	hasAdSpend: false,
+	hasAvatar: false,
+	hasBrowser: false,
+	hasCampaigns: false,
+	hasIOSDevices: false,
+	hasLocation: false,
+	isAnonymous: true,
+	hasAndroidDevices: false,
+	hasDesktopDevices: false,
+	writeToDisk: true,
+	concurrency: 1,
+
+	events: [
+		{
+			// ~200 string/number/bool/list properties = fat rows approaching 1MB
+			event: "mega_row",
+			weight: 5,
+			properties: {
+				...manyProperties(250),
+				// also throw in some big array-of-object columns
+				cart_items: bigArrayOfObjects(60),
+				order_history: bigArrayOfObjects(80),
+			}
+		},
+		{
+			// single event focused on huge array-of-object columns (well over 8KB each)
+			event: "array_bomb",
+			weight: 5,
+			properties: {
+				massive_list_a: bigArrayOfObjects(150),
+				massive_list_b: bigArrayOfObjects(150),
+				massive_list_c: bigArrayOfObjects(100),
+				some_prop: ["foo", "bar", "baz"],
+				another_prop: () => integer(1, 1000),
+			}
+		},
+		{
+			// moderate event with a mix of oversized columns
+			event: "chonky_boi",
+			weight: 3,
+			properties: {
+				...manyProperties(100),
+				nested_blob: () => {
+					const obj = {};
+					for (let i = 0; i < 50; i++) {
+						obj[`key_${i}`] = {
+							value: loremChunk(200),
+							items: Array.from({ length: 20 }, (_, j) => ({
+								id: j,
+								data: loremChunk(100),
+								flag: Math.random() > 0.5
+							}))
+						};
+					}
+					return obj;
+				},
+				product_catalog: bigArrayOfObjects(120),
+			}
+		},
+		{
+			// normal-ish event for contrast
+			event: "smol_event",
+			weight: 2,
+			properties: {
+				color: ["red", "blue", "green"],
+				count: () => integer(1, 10),
+			}
+		}
+	],
+
+	superProps: {},
+	userProps: {
+		name: () => `user_${Math.random().toString(36).slice(2, 8)}`,
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	lookupTables: [],
+	groupKeys: [],
+	groupProps: {},
+
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+export default config;

package/dungeons/uday-schema.json

@@ -0,0 +1,220 @@
+{
+  "events": [
+    {
+      "event": "account created",
+      "weight": 0,
+      "isFirstEvent": true,
+      "properties": {
+        "signup_source": ["organic", "referral", "demo_request", "product_hunt", "g2_review"],
+        "initial_plan": ["free_trial", "free_trial", "free_trial", "growth", "enterprise"]
+      }
+    },
+    {
+      "event": "agent deployed",
+      "weight": 3,
+      "properties": {
+        "agent_type": ["pre_call_researcher", "meeting_assistant", "follow_up_writer", "crm_updater", "pipeline_monitor", "outreach_agent"],
+        "configuration_time_min": {"$range": [2, 45]}
+      }
+    },
+    {
+      "event": "meeting joined",
+      "weight": 5,
+      "properties": {
+        "meeting_platform": ["zoom", "zoom", "google_meet", "google_meet", "teams", "webex"],
+        "meeting_type": ["discovery", "demo", "negotiation", "check_in", "onboarding", "qbr"],
+        "participants": {"$range": [2, 12]},
+        "duration_min": {"$range": [10, 90]}
+      }
+    },
+    {
+      "event": "meeting summary generated",
+      "weight": 5,
+      "properties": {
+        "summary_length_words": {"$range": [100, 800]},
+        "action_items_count": {"$range": [0, 8]},
+        "quality_score": {"$range": [50, 95]}
+      }
+    },
+    {
+      "event": "crm auto-updated",
+      "weight": 8,
+      "properties": {
+        "crm_platform": ["hubspot", "hubspot", "salesforce", "salesforce", "pipedrive"],
+        "fields_updated": {"$range": [1, 6]},
+        "update_type": ["contact_info", "deal_stage", "activity_log", "meeting_notes", "next_steps"]
+      }
+    },
+    {
+      "event": "follow-up drafted",
+      "weight": 4,
+      "properties": {
+        "personalization_score": {"$range": [30, 98]},
+        "template_used": ["post_meeting", "deal_nudge", "re_engagement", "intro", "proposal"],
+        "word_count": {"$range": [50, 400]}
+      }
+    },
+    {
+      "event": "follow-up sent",
+      "weight": 3,
+      "properties": {
+        "channel": ["email", "email", "email", "linkedin", "slack"],
+        "personalization_score": {"$range": [30, 98]},
+        "time_to_send_hr": {"$range": [0, 48]},
+        "email_outcome": ["pending", "pending", "pending", "opened", "opened", "clicked", "bounced", "replied"]
+      }
+    },
+    {
+      "event": "pipeline alert received",
+      "weight": 4,
+      "properties": {
+        "alert_type": ["deal_stalling", "champion_left", "competitor_mentioned", "no_activity", "budget_risk"],
+        "severity": ["low", "medium", "high", "critical"],
+        "deal_value": {"$range": [5000, 500000]}
+      }
+    },
+    {
+      "event": "deal stage changed",
+      "weight": 5,
+      "properties": {
+        "from_stage": ["lead", "qualified", "proposal", "negotiation", "verbal_commit"],
+        "to_stage": ["qualified", "proposal", "negotiation", "verbal_commit", "closed_won", "closed_lost"],
+        "days_in_previous_stage": {"$range": [1, 60]}
+      }
+    },
+    {
+      "event": "insight generated",
+      "weight": 6,
+      "properties": {
+        "insight_type": ["deal_risk", "buyer_intent", "competitive_intel", "engagement_trend", "revenue_forecast", "coaching_opportunity"],
+        "confidence_score": {"$range": [55, 98]},
+        "data_sources_used": {"$range": [1, 5]}
+      }
+    },
+    {
+      "event": "agent feedback",
+      "weight": 4,
+      "properties": {
+        "rating": [1, 2, 3, 3, 4, 4, 4, 5, 5, 5],
+        "feedback_type": ["accuracy", "relevance", "timeliness", "completeness", "tone"],
+        "agent_type": ["meeting_assistant", "follow_up_writer", "crm_updater", "pre_call_researcher"]
+      }
+    },
+    {
+      "event": "dashboard viewed",
+      "weight": 7,
+      "properties": {
+        "dashboard_type": ["pipeline_overview", "team_performance", "ai_agent_stats", "revenue_forecast", "activity_feed"],
+        "time_spent_sec": {"$range": [10, 300]}
+      }
+    },
+    {
+      "event": "integration connected",
+      "weight": 1,
+      "properties": {
+        "integration_name": ["hubspot", "salesforce", "slack", "google_calendar", "zoom", "linkedin", "gmail", "outlook"],
+        "setup_time_min": {"$range": [2, 30]}
+      }
+    },
+    {
+      "event": "search performed",
+      "weight": 3,
+      "properties": {
+        "search_type": ["deals", "contacts", "meetings", "insights", "activities"],
+        "results_count": {"$range": [0, 50]},
+        "clicked_result": [true, true, true, false]
+      }
+    },
+    {
+      "event": "workflow created",
+      "weight": 2,
+      "properties": {
+        "workflow_type": ["meeting_prep", "post_meeting", "deal_update", "weekly_digest", "pipeline_review"],
+        "steps_count": {"$range": [2, 8]},
+        "trigger_type": ["time_based", "event_based", "manual"]
+      }
+    },
+    {
+      "event": "contact enriched",
+      "weight": 5,
+      "properties": {
+        "enrichment_source": ["linkedin", "company_website", "crm_history", "email_threads", "meeting_notes"],
+        "fields_enriched": {"$range": [2, 12]},
+        "confidence_level": ["high", "high", "medium", "medium", "low"]
+      }
+    },
+    {
+      "event": "deal created",
+      "weight": 2,
+      "properties": {
+        "deal_source": ["inbound", "outbound", "referral", "upsell", "expansion"],
+        "estimated_value": {"$range": [5000, 250000]},
+        "close_date_days": {"$range": [14, 120]}
+      }
+    },
+    {
+      "event": "coaching tip viewed",
+      "weight": 3,
+      "properties": {
+        "tip_category": ["objection_handling", "discovery_questions", "closing_techniques", "rapport_building", "negotiation"],
+        "tip_relevance_score": {"$range": [40, 98]},
+        "applied": [true, false, false, false]
+      }
+    }
+  ],
+  "superProps": {
+    "subscription_tier": ["starter", "growth", "growth", "enterprise"],
+    "team_role": ["sales_rep", "sales_rep", "sales_rep", "sales_manager", "rev_ops", "marketing"],
+    "ai_agent_version": ["v1.0", "v1.0", "v1.5", "v1.5", "v1.5", "v2.0", "v2.0", "v2.0"]
+  },
+  "userProps": {
+    "company_size": ["SMB", "mid_market", "enterprise"],
+    "industry": ["SaaS", "fintech", "healthcare", "e_commerce", "consulting", "manufacturing"],
+    "deals_in_pipeline": {"$range": [2, 50]},
+    "onboarding_cohort": ["Q1_2025", "Q2_2025", "Q3_2025", "Q4_2025"],
+    "integrations_connected": {"$range": [1, 8]}
+  },
+  "funnels": [
+    {
+      "name": "Onboarding",
+      "sequence": ["account created", "integration connected", "agent deployed", "meeting joined"],
+      "isFirstFunnel": true,
+      "conversionRate": 65,
+      "timeToConvert": 48,
+      "order": "sequential",
+      "weight": 1
+    },
+    {
+      "name": "Meeting to Action",
+      "sequence": ["meeting joined", "meeting summary generated", "follow-up drafted", "follow-up sent"],
+      "conversionRate": 70,
+      "timeToConvert": 4,
+      "order": "sequential",
+      "weight": 8
+    },
+    {
+      "name": "Pipeline Management",
+      "sequence": ["pipeline alert received", "dashboard viewed", "deal stage changed"],
+      "conversionRate": 45,
+      "timeToConvert": 24,
+      "order": "sequential",
+      "weight": 5
+    },
+    {
+      "name": "AI Adoption",
+      "sequence": ["agent deployed", "insight generated", "agent feedback"],
+      "conversionRate": 55,
+      "timeToConvert": 72,
+      "order": "sequential",
+      "weight": 4
+    },
+    {
+      "name": "Value Realization",
+      "sequence": ["dashboard viewed", "workflow created", "agent deployed"],
+      "conversionRate": 40,
+      "timeToConvert": 96,
+      "order": "sequential",
+      "weight": 3
+    }
+  ]
+}

package/dungeons/userAgent.js

@@ -1,10 +1,10 @@
 
 const SEED = "my-seed";
 import dayjs from 'dayjs';
-import utc from 'dayjs/plugin/utc';
+import utc from 'dayjs/plugin/utc.js';
 dayjs.extend(utc);
 import 'dotenv/config';
-import * as u from '../lib/utils/utils.js';
+import * as u from "../lib/utils/utils.js";
 import * as v from 'ak-tools';
 const chance = u.initChance(SEED);
 const num_users = 1000;
@@ -14,7 +14,7 @@ const days = 30;
 
 /** @type {Config} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 100,
@@ -33,6 +33,7 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
+	makeChart: false,
 
 	batchSize: 500_000,
 	concurrency: 500,

package/index.js

@@ -21,8 +21,6 @@ import { StorageManager } from './lib/core/storage.js';
 // Orchestrators  
 import { userLoop } from './lib/orchestrators/user-loop.js';
 import { sendToMixpanel } from './lib/orchestrators/mixpanel-sender.js';
-import { handleCloudFunctionEntry } from './lib/orchestrators/worker-manager.js';
-
 // Generators
 import { makeAdSpend } from './lib/generators/adspend.js';
 import { makeMirror } from './lib/generators/mirror.js';
@@ -33,9 +31,9 @@ import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 // External dependencies
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
-import functions from '@google-cloud/functions-framework';
-import { timer, sLog } from 'ak-tools';
+import { timer } from 'ak-tools';
 import { existsSync } from 'fs';
+import { dataLogger as logger } from './lib/utils/logger.js';
 
 // Initialize dayjs and time constants
 dayjs.extend(utc);
@@ -134,19 +132,23 @@ async function main(config) {
 
 	}
 
-	if (config.verbose) console.log(`\n🔧 Configuring dungeon with seed: ${config.seed}`);
+	if (config.verbose) logger.info({ seed: config.seed }, 'Configuring dungeon');
 	let validatedConfig;
 	try {
 		// Step 1: Validate and enrich configuration
 		validatedConfig = validateDungeonConfig(config);
-		
+
+		// Update FIXED_BEGIN based on configured numDays
+		const configNumDays = validatedConfig.numDays || 30;
+		global.FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(configNumDays, 'd').unix();
+
 		// Step 1.5: Display configuration summary (CLI mode only)
 		if (isCLI && validatedConfig.verbose) {
 			displayConfigurationSummary(validatedConfig);
 		}
 
 		// Step 2: Create context with validated config
-		const context = createContext(validatedConfig, null, isCLI);
+		const context = createContext(validatedConfig);
 
 		// Step 3: Initialize storage containers
 		const storageManager = new StorageManager(context);
@@ -160,7 +162,7 @@ async function main(config) {
 			await generateAdSpendData(context);
 		}
 
-		if (context.config.verbose) console.log(`\n🔄 Starting user and event generation...\n`);
+		if (context.config.verbose) logger.info('Starting user and event generation...');
 		// Step 5: Main user and event generation
 		await userLoop(context);
 
@@ -184,27 +186,27 @@ async function main(config) {
 			await makeMirror(context);
 		}
 
-		if (context.config.verbose) console.log(`\n✅ Data generation completed successfully!\n`);
+		if (context.config.verbose) logger.info('Data generation completed successfully');
 
 		// ! DATA GENERATION ENDS HERE
 
-		// Step 10: Send to Mixpanel (if token provided)
-		// IMPORTANT: Must happen BEFORE flushing to disk, because flush() clears the arrays
-		let importResults;
-		if (validatedConfig.token) {
-			importResults = await sendToMixpanel(context);
-		}
-
-		// Step 11: Flush lookup tables to disk (always as CSVs)
+		// Step 10: Flush lookup tables to disk (always as CSVs)
 		if (validatedConfig.writeToDisk) {
 			await flushLookupTablesToDisk(storage, validatedConfig);
 		}
 
-		// Step 12: Flush other storage containers to disk (if writeToDisk enabled)
+		// Step 11: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
 
+		// Step 12: Send to Mixpanel (if token provided)
+		// Now happens AFTER disk flush so batch files are available for import
+		let importResults;
+		if (validatedConfig.token) {
+			importResults = await sendToMixpanel(context);
+		}
+
 		// Step 13: Compile results
 		jobTimer.stop(false);
 		const { start, end, delta, human } = jobTimer.report(false);
@@ -222,12 +224,7 @@ async function main(config) {
 		};
 
 	} catch (error) {
-		if (validatedConfig?.verbose) {
-			console.error(`\n❌ Error: ${error.message}\n`);
-			console.error(error.stack);
-		} else {
-			sLog("Main execution error", { error: error.message, stack: error.stack }, "ERROR");
-		}
+		logger.error({ err: error }, `Error: ${error.message}`);
 		throw error;
 	}
 }
@@ -260,8 +257,8 @@ async function generateGroupProfiles(context) {
 	const { config, storage } = context;
 	const { groupKeys, groupProps = {} } = config;
 
-	if (context.isCLI() || config.verbose) {
-		console.log('\n👥 Generating group profiles...');
+	if (config.verbose) {
+		logger.info('Generating group profiles...');
 	}
 
 	for (let i = 0; i < groupKeys.length; i++) {
@@ -273,8 +270,8 @@ async function generateGroupProfiles(context) {
 			continue;
 		}
 
-		if (context.isCLI() || config.verbose) {
-			console.log(`   Creating ${groupCount.toLocaleString()} ${groupKey} profiles...`);
+		if (config.verbose) {
+			logger.info({ groupKey, groupCount }, `Creating ${groupCount.toLocaleString()} ${groupKey} profiles...`);
 		}
 
 		// Get group-specific props if available
@@ -282,15 +279,15 @@ async function generateGroupProfiles(context) {
 
 		for (let j = 0; j < groupCount; j++) {
 			const groupProfile = await makeGroupProfile(context, groupKey, specificGroupProps, {
-				[groupKey]: `${groupKey}_${j + 1}`
+				[groupKey]: String(j + 1)
 			});
 
 			await groupContainer.hookPush(groupProfile);
 		}
 	}
 
-	if (context.isCLI() || config.verbose) {
-		console.log('✅ Group profiles generated successfully');
+	if (config.verbose) {
+		logger.info('Group profiles generated successfully');
 	}
 }
 
@@ -302,8 +299,8 @@ async function generateLookupTables(context) {
 	const { config, storage } = context;
 	const { lookupTables } = config;
 
-	if (context.isCLI() || config.verbose) {
-		console.log('\n🔍 Generating lookup tables...');
+	if (config.verbose) {
+		logger.info('Generating lookup tables...');
 	}
 
 	for (let i = 0; i < lookupTables.length; i++) {
@@ -316,8 +313,8 @@ async function generateLookupTables(context) {
 			continue;
 		}
 
-		if (context.isCLI() || config.verbose) {
-			console.log(`   Creating ${entries.toLocaleString()} ${key} lookup entries...`);
+		if (config.verbose) {
+			logger.info({ key, entries }, `Creating ${entries.toLocaleString()} ${key} lookup entries...`);
 		}
 
 		for (let j = 0; j < entries; j++) {
@@ -330,8 +327,8 @@ async function generateLookupTables(context) {
 		}
 	}
 
-	if (context.isCLI() || config.verbose) {
-		console.log('✅ Lookup tables generated successfully');
+	if (config.verbose) {
+		logger.info('Lookup tables generated successfully');
 	}
 }
 
@@ -343,8 +340,8 @@ async function generateGroupSCDs(context) {
 	const { config, storage } = context;
 	const { scdProps, groupKeys } = config;
 
-	if (context.isCLI() || config.verbose) {
-		console.log('\n📊 Generating group SCDs...');
+	if (config.verbose) {
+		logger.info('Generating group SCDs...');
 	}
 
 	// Import utilities and generators
@@ -366,13 +363,13 @@ async function generateGroupSCDs(context) {
 			continue; // No SCDs for this group type
 		}
 
-		if (context.isCLI() || config.verbose) {
-			console.log(`   Generating SCDs for ${groupCount.toLocaleString()} ${groupKey} entities...`);
+		if (config.verbose) {
+			logger.info({ groupKey, groupCount }, `Generating SCDs for ${groupCount.toLocaleString()} ${groupKey} entities...`);
 		}
 
 		// Generate SCDs for each group entity
 		for (let i = 0; i < groupCount; i++) {
-			const groupId = `${groupKey}_${i + 1}`;
+			const groupId = String(i + 1);
 
 			// Generate SCDs for this group entity
 			for (const [scdKey, scdConfig] of Object.entries(groupSpecificSCDs)) {
@@ -408,8 +405,8 @@ async function generateGroupSCDs(context) {
 		}
 	}
 
-	if (context.isCLI() || config.verbose) {
-		console.log('✅ Group SCDs generated successfully');
+	if (config.verbose) {
+		logger.info('Group SCDs generated successfully');
 	}
 }
 
@@ -566,16 +563,6 @@ function extractStorageData(storage) {
 	};
 }
 
-// Cloud Functions setup
-functions.http('entry', async (req, res) => {
-	await handleCloudFunctionEntry(req, res, main);
-});
-
 // ES Module export
 export default main;
 
-// CommonJS compatibility
-if (typeof module !== 'undefined' && module.exports) {
-	module.exports = main;
-}
-