make-mp-data 3.0.2 → 3.0.4

package/dungeons/streaming.js

@@ -44,10 +44,9 @@ const config = {
 	hasAdSpend: false,
 
 	hasAvatar: false,
-	makeChart: false,
 
 	batchSize: 2_500_000,
-	concurrency: 10,
+	concurrency: 1,
 	writeToDisk: false,
 
 	funnels: [],
@@ -66,7 +65,7 @@ const config = {
 				]),
 				"watch time": u.weighNumRange(1, 65, .89, 100),
 
-				"category": u.pickAWinner([
+				"category": [
 					"comedy",
 					"educational",
 					"music",
@@ -74,15 +73,15 @@ const config = {
 					"news",
 					"gaming",
 					"travel",
-				]),
-				quality: u.pickAWinner([
+				],
+				quality: [
 					"240p",
 					"360p",
 					"480p",
 					"720p",
 					"1080p",
 					"4k",
-				], 4),
+				],
 				autoplay: [
 					true,
 					false,
@@ -121,13 +120,13 @@ const config = {
 			weight: 3,
 			properties: {
 				video_id: u.pickAWinner(videoIds),
-				"share network": u.pickAWinner([
+				"share network": [
 					"facebook",
 					"twitter",
 					"reddit",
 					"email",
 					"whatsapp",
-				]),
+				],
 			},
 		},
 		{
@@ -175,18 +174,18 @@ const config = {
 			event: "create playlist",
 			weight: 4,
 			properties: {
-				"play list name": u.pickAWinner([
+				"play list name": [
 					"favorites",
 					"watch later",
 					"my music",
 					"funny videos",
 					"educational",
-				]),
-				privacy: u.pickAWinner([
+				],
+				privacy: [
 					"public",
 					"private",
 					"unlisted",
-				]),
+				],
 			},
 		},
 		{
@@ -205,11 +204,11 @@ const config = {
 			event: "account login",
 			weight: 9,
 			properties: {
-				"log in method": u.pickAWinner([
+				"log in method": [
 					"email",
 					"google",
 					"facebook",
-				]),
+				],
 				success: [
 					true,
 					false,
@@ -233,11 +232,11 @@ const config = {
 		}
 	],
 	superProps: {
-		platform: u.pickAWinner([
+		platform: [
 			"web",
 			"ios",
 			"android",
-		]),
+		],
 		network_type: [
 			"wifi",
 			"cellular",
@@ -257,13 +256,13 @@ const config = {
 			"45-54",
 			"55+",
 		],
-		preferred_genre: u.pickAWinner([
+		preferred_genre: [
 			"comedy",
 			"action",
 			"drama",
 			"sci-fi",
 			"horror",
-		]),
+		],
 		upload_count: [
 			0,
 			1,

package/dungeons/student-teacher.js

@@ -34,7 +34,6 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
-	makeChart: false,
 
 	batchSize: 2_500_000,
 	concurrency: 1,

package/dungeons/text-generation.js

@@ -344,7 +344,6 @@ const dungeon = {
 	isAnonymous: false,
 	hasAdSpend: false,	
 	hasAvatar: false,
-	makeChart: false,
 
 	batchSize: 2_500_000,
 	concurrency: 1,

package/dungeons/user-agent.js

@@ -33,10 +33,9 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
-	makeChart: false,
 
 	batchSize: 500_000,
-	concurrency: 500,
+	concurrency: 1,
 	writeToDisk: false,
 
 	funnels: [],

package/index.js

@@ -27,6 +27,7 @@ import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 
 // Utilities
+import { initChance } from './lib/utils/utils.js';
 
 // External dependencies
 import dayjs from "dayjs";
@@ -138,6 +139,12 @@ async function main(config) {
 		// Step 1: Validate and enrich configuration
 		validatedConfig = validateDungeonConfig(config);
 
+		// Ensure seeded RNG is initialized (dungeons do this at module scope,
+		// but npm-module consumers pass seed via config object)
+		if (validatedConfig.seed) {
+			initChance(validatedConfig.seed);
+		}
+
 		// Update FIXED_BEGIN based on configured numDays
 		const configNumDays = validatedConfig.numDays || 30;
 		global.FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(configNumDays, 'd').unix();
@@ -237,8 +244,11 @@ async function generateAdSpendData(context) {
 	const { config, storage } = context;
 	const { numDays } = config;
 
+	const timeShift = context.TIME_SHIFT_SECONDS;
 	for (let day = 0; day < numDays; day++) {
-		const targetDay = dayjs.unix(global.FIXED_BEGIN).add(day, 'day').toISOString();
+		const fixedDay = dayjs.unix(global.FIXED_BEGIN).add(day, 'day').unix();
+		const shiftedDay = Math.min(fixedDay + timeShift, context.MAX_TIME);
+		const targetDay = dayjs.unix(shiftedDay).toISOString();
 		const adSpendEvents = await makeAdSpend(context, targetDay);
 
 		if (adSpendEvents.length > 0) {
@@ -266,7 +276,7 @@ async function generateGroupProfiles(context) {
 		const groupContainer = storage.groupProfilesData[i];
 
 		if (!groupContainer) {
-			console.warn(`Warning: No storage container found for group key: ${groupKey}`);
+			if (config.verbose) console.warn(`Warning: No storage container found for group key: ${groupKey}`);
 			continue;
 		}
 
@@ -309,7 +319,7 @@ async function generateLookupTables(context) {
 		const lookupContainer = storage.lookupTableData[i];
 
 		if (!lookupContainer) {
-			console.warn(`Warning: No storage container found for lookup table: ${key}`);
+			if (config.verbose) console.warn(`Warning: No storage container found for lookup table: ${key}`);
 			continue;
 		}
 
@@ -378,16 +388,19 @@ async function generateGroupSCDs(context) {
 
 				// Use a base time for the group entity (similar to user creation time)
 				const baseTime = context.FIXED_BEGIN || context.FIXED_NOW;
-				const changes = await makeSCD(context, scdConfig, scdKey, groupId, mutations, baseTime);
+				let changes = await makeSCD(context, scdConfig, scdKey, groupId, mutations, baseTime);
 
 				// Apply hook if configured
 				if (config.hook) {
-					await config.hook(changes, "scd-pre", {
+					const hookResult = await config.hook(changes, "scd-pre", {
 						type: 'group',
 						groupKey,
 						scd: { [scdKey]: scdConfig },
 						config
 					});
+					if (Array.isArray(hookResult)) {
+						changes = hookResult;
+					}
 				}
 
 				// Store SCDs in the appropriate SCD table
@@ -538,7 +551,6 @@ async function extractFileInfo(storage, config) {
 			}
 		} catch (error) {
 			// If scanning fails, just return empty array
-			console.warn('Warning: Could not scan data directory for files:', error.message);
 		}
 	}
 

package/lib/core/config-validator.js

@@ -106,8 +106,7 @@ export function validateDungeonConfig(config) {
 		token = null,
 		region = "US",
 		writeToDisk = false,
-		verbose = true,
-		makeChart = false,
+		verbose = false,
 		soup = {},
 		hook = (record) => record,
 		hasAdSpend = false,
@@ -167,15 +166,17 @@ export function validateDungeonConfig(config) {
 				throw new Error('Hook string did not evaluate to a function');
 			}
 		} catch (error) {
-			console.warn(`\u26a0\ufe0f Failed to convert hook string to function: ${error.message}`);
-			console.warn('Using default pass-through hook');
+			if (config.verbose !== false) {
+				console.warn(`\u26a0\ufe0f Failed to convert hook string to function: ${error.message}`);
+				console.warn('Using default pass-through hook');
+			}
 			hook = (record) => record;
 		}
 	}
 
 	// Ensure hook is a function
 	if (typeof hook !== 'function') {
-		console.warn('\u26a0\ufe0f Hook is not a function, using default pass-through hook');
+		if (config.verbose !== false) console.warn('\u26a0\ufe0f Hook is not a function, using default pass-through hook');
 		hook = (record) => record;
 	}
 
@@ -187,6 +188,17 @@ export function validateDungeonConfig(config) {
 		events = events.map(e => ({ event: /** @type {string} */ (e) }));
 	}
 
+	// Validate: if every user is born in dataset, we need either isFirstEvent or isFirstFunnel
+	const percentBorn = config.percentUsersBornInDataset ?? 15;
+	const hasFirstEvent = events.some(e => e.isFirstEvent);
+	const hasFirstFunnel = funnels.some(f => f.isFirstFunnel);
+	if (percentBorn >= 100 && !hasFirstEvent && !hasFirstFunnel) {
+		throw new Error(
+			"percentUsersBornInDataset is 100% but no event has isFirstEvent and no funnel has isFirstFunnel. " +
+			"Either add isFirstEvent to an event, add a first funnel, or lower percentUsersBornInDataset."
+		);
+	}
+
 	// Handle funnel inference
 	if (alsoInferFunnels) {
 		const inferredFunnels = inferFunnels(events);
@@ -202,6 +214,7 @@ export function validateDungeonConfig(config) {
 
 	const eventsNotInFunnels = events
 		.filter(e => !e.isFirstEvent)
+		.filter(e => !e.isStrictEvent)
 		.filter(e => !eventContainedInFunnels.includes(e.event))
 		.map(e => e.event);
 
@@ -283,36 +296,12 @@ export function validateDungeonConfig(config) {
 		hasDesktopDevices,
 		hasIOSDevices,
 		name,
-		makeChart,
 		strictEventCount
 	};
 
 	return validatedConfig;
 }
 
-/**
- * Validates configuration for required fields
- * @param {Object} config - Configuration to validate
- * @throws {Error} If required fields are missing
- */
-/**
- * Validates required configuration parameters
- * @param {Dungeon} config - Configuration object to validate
- * @returns {boolean} True if validation passes
- */
-export function validateRequiredConfig(config) {
-	if (!config) {
-		throw new Error("Configuration is required");
-	}
-
-	if (typeof config !== 'object') {
-		throw new Error("Configuration must be an object");
-	}
-
-	// Could add more specific validation here
-	return true;
-}
-
 /**
  * Transforms SCD properties to regular user/group properties when service account credentials are missing
  * ONLY applies to UI jobs - programmatic usage always generates SCD files
@@ -350,7 +339,7 @@ function transformSCDPropsWithoutCredentials(config) {
 	}
 
 	// UI job without credentials - convert SCD props to regular props
-	console.log('\u26a0\ufe0f  Service account credentials missing - converting SCD properties to static properties');
+	if (config.verbose !== false) console.log('\u26a0\ufe0f  Service account credentials missing - converting SCD properties to static properties');
 
 	// Ensure userProps and groupProps exist
 	if (!config.userProps) config.userProps = {};
@@ -369,20 +358,20 @@ function transformSCDPropsWithoutCredentials(config) {
 		if (type === "user") {
 			// Add to userProps
 			config.userProps[propKey] = values;
-			console.log(`  \u2713 Converted user SCD property: ${propKey}`);
+			if (config.verbose !== false) console.log(`  \u2713 Converted user SCD property: ${propKey}`);
 		} else {
 			// Add to groupProps for the specific group type
 			if (!config.groupProps[type]) {
 				config.groupProps[type] = {};
 			}
 			config.groupProps[type][propKey] = values;
-			console.log(`  \u2713 Converted group SCD property: ${propKey} (${type})`);
+			if (config.verbose !== false) console.log(`  \u2713 Converted group SCD property: ${propKey} (${type})`);
 		}
 	}
 
 	// Clear out scdProps since we've converted everything
 	config.scdProps = {};
-	console.log('\u2713 SCD properties converted to static properties\n');
+	if (config.verbose !== false) console.log('\u2713 SCD properties converted to static properties\n');
 }
 
 export { inferFunnels, transformSCDPropsWithoutCredentials };

package/lib/core/context.js

@@ -149,12 +149,12 @@ export function createContext(config, storage = null, _unusedCliMode = null) {
 
 		// Time helper methods
 		getTimeShift() {
-			const actualNow = dayjs().add(1, "day");
+			const actualNow = dayjs().subtract(1, "hour");
 			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "seconds");
 		},
 
 		getDaysShift() {
-			const actualNow = dayjs().add(1, "day");
+			const actualNow = dayjs().subtract(1, "hour");
 			return actualNow.diff(dayjs.unix(this.FIXED_NOW), "days");
 		},
 
@@ -164,9 +164,12 @@ export function createContext(config, storage = null, _unusedCliMode = null) {
 
 		// PERFORMANCE: Pre-calculated time shift (instead of calculating per-event)
 		TIME_SHIFT_SECONDS: (() => {
-			const actualNow = dayjs().add(1, "day");
+			const actualNow = dayjs().subtract(1, "hour");
 			return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
 		})(),
+
+		// Max timestamp (unix seconds) — clamp here to prevent future events
+		MAX_TIME: dayjs().unix(),
 	};
 
 	return context;

package/lib/generators/events.js

@@ -52,7 +52,9 @@ export async function makeEvent(
     const chance = u.getChance();
     
     // Extract soup configuration for time distribution
-    const { mean = 0, deviation = 2, peaks = 5 } = config.soup || {};
+    // Dynamic peaks: one per week for long ranges, minimum 5
+    const defaultPeaks = Math.max(5, Math.ceil((config.numDays || 30) / 7));
+    const { mean = 0, deviation = 2, peaks = defaultPeaks } = config.soup || {};
     
     // Extract feature flags from config
     const {
@@ -95,18 +97,19 @@ export async function makeEvent(
 
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
+        let shiftedTimestamp;
         if (isFirstEvent) {
-            // PERFORMANCE: Direct numeric calculation instead of dayjs object creation
-            const shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
-            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
+            shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
         } else {
-            // Get time from TimeSoup (returns ISO string) and apply precomputed time shift
-            const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
-            // PERFORMANCE: Parse ISO directly to milliseconds, add shift, convert back to ISO with one dayjs call
-            const soupTimestamp = new Date(soupTime).getTime() / 1000; // Convert to unix seconds
-            const shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
-            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
+            // TimeSoup returns unix seconds; shift and convert to ISO once
+            const soupTimestamp = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
         }
+        // Drop events that would land in the future (Mixpanel rewrites these to "now", causing pile-ups)
+        if (shiftedTimestamp > context.MAX_TIME) {
+            eventTemplate._drop = true;
+        }
+        eventTemplate.time = dayjs.unix(Math.min(shiftedTimestamp, context.MAX_TIME)).toISOString();
     }
 
     // Add anonymous and session identifiers

package/lib/generators/funnels.js

@@ -131,10 +131,9 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
 		numStepsUserWillTake
 	);
 
-	// Add session start event if configured
+	// Add session start event if configured (clone to avoid mutating shared config)
 	if (sessionStartEvents.length) {
-		const sessionStartEvent = chance.pickone(sessionStartEvents);
-		sessionStartEvent.relativeTimeMs = -15000; // 15 seconds before funnel
+		const sessionStartEvent = { ...chance.pickone(sessionStartEvents), relativeTimeMs: -15000 };
 		funnelEventsWithTiming.push(sessionStartEvent);
 	}
 
@@ -386,7 +385,11 @@ async function generateFunnelEvents(
 		}
 
 		try {
-			const computedTime = dayjs(funnelStartTime).add(event.relativeTimeMs, "milliseconds");
+			let computedTime = dayjs(funnelStartTime).add(event.relativeTimeMs, "milliseconds");
+			// Drop events that would land in the future
+			if (context.MAX_TIME && computedTime.unix() > context.MAX_TIME) {
+				newEvent._drop = true;
+			}
 			if (computedTime.isValid()) {
 				newEvent.time = computedTime.toISOString();
 			}

package/lib/generators/scd.js

@@ -41,19 +41,22 @@ export async function makeSCD(context, scdProp, scdKey, distinct_id, mutations,
     } = scdProp;
     
     // Return empty array if no values provided
-    if (JSON.stringify(values) === "{}" || JSON.stringify(values) === "[]") {
+    if (!values || (Array.isArray(values) && values.length === 0) ||
+        (typeof values === 'object' && Object.keys(values).length === 0)) {
         return [];
     }
-    
+
     const scdEntries = [];
     let lastInserted = dayjs(created);
-    const deltaDays = dayjs().diff(lastInserted, "day");
+    const now = dayjs();
+    const deltaDays = now.diff(lastInserted, "day");
     const uuidKeyName = type === 'user' ? 'distinct_id' : type;
+    let lastStartTime = null; // Track for monotonic ordering
 
     for (let i = 0; i < mutations; i++) {
         // Stop if we've reached the current time
-        if (lastInserted.isAfter(dayjs())) break;
-        
+        if (lastInserted.isAfter(now)) break;
+
         // Create profile with the SCD property
         const scd = await makeProfile(context, { [scdKey]: values }, { [uuidKeyName]: distinct_id });
 
@@ -78,6 +81,9 @@ export async function makeSCD(context, scdProp, scdKey, distinct_id, mutations,
                 case "month":
                     scdEntry.startTime = lastInserted.add(1, "month").startOf('month').toISOString();
                     break;
+                case "year":
+                    scdEntry.startTime = lastInserted.add(1, "year").startOf('year').toISOString();
+                    break;
             }
         }
 
@@ -85,28 +91,34 @@ export async function makeSCD(context, scdProp, scdKey, distinct_id, mutations,
             scdEntry.startTime = lastInserted.toISOString();
         }
 
+        // Enforce monotonic ordering — startTime must advance
+        if (lastStartTime && scdEntry.startTime && scdEntry.startTime <= lastStartTime) {
+            scdEntry.startTime = dayjs(lastStartTime).add(1, "second").toISOString();
+        }
+        lastStartTime = scdEntry.startTime;
+
         // Set insert time (slightly after start time)
-        const insertTime = lastInserted.add(u.integer(1, 9000), "seconds");
+        const insertTime = dayjs(scdEntry.startTime).add(u.integer(1, 9000), "seconds");
         scdEntry.insertTime = insertTime.toISOString();
 
         // Set the time field for Mixpanel SCD import (uses startTime)
         scdEntry.time = scdEntry.startTime;
 
         // Only add entry if all required properties are set
-        if (scdEntry.hasOwnProperty('insertTime') && scdEntry.hasOwnProperty('startTime') && scdEntry.hasOwnProperty('time')) {
+        if (scdEntry.startTime && scdEntry.insertTime && scdEntry.time) {
             scdEntries.push(scdEntry);
         }
 
-        // Advance time for next entry
-        lastInserted = lastInserted
-            .add(u.integer(0, deltaDays), "day")
-            .subtract(u.integer(1, 9000), "seconds");
+        // Advance time for next entry — scale step size to available range
+        if (deltaDays > 1) {
+            const advance = Math.max(1, u.integer(0, Math.ceil(deltaDays / mutations)));
+            lastInserted = dayjs(scdEntry.startTime).add(advance, "day");
+        } else {
+            // Short range: advance by hours instead of days
+            const advance = Math.max(1, u.integer(1, 12));
+            lastInserted = dayjs(scdEntry.startTime).add(advance, "hour");
+        }
     }
 
-    // De-duplicate on startTime
-    const deduped = scdEntries.filter((entry, index, self) =>
-        index === self.findIndex((t) => t.startTime === entry.startTime)
-    );
-    
-    return deduped;
+    return scdEntries;
 }

package/lib/generators/text.js

@@ -27,23 +27,29 @@ import seedrandom from 'seedrandom';
 import crypto from 'crypto';
 import SentimentPkg from 'sentiment';
 import { PHRASE_BANK, GENERATION_PATTERNS, ORGANIC_PATTERNS } from '../templates/phrases.js';
+import { getChance } from '../utils/utils.js';
 
 const Sentiment = typeof SentimentPkg === 'function' ? SentimentPkg : SentimentPkg.default;
 const sentiment = new Sentiment();
 
 // ============= Helper Functions =============
 
+function seededRandom() {
+    const c = getChance();
+    return c.floating({ min: 0, max: 1 });
+}
+
 function chance(probability) {
-    return Math.random() < probability;
+    return seededRandom() < probability;
 }
 
 function pick(array) {
-    return array[Math.floor(Math.random() * array.length)];
+    return array[Math.floor(seededRandom() * array.length)];
 }
 
 function pickWeighted(items, weights) {
     const total = weights.reduce((a, b) => a + b, 0);
-    let random = Math.random() * total;
+    let random = seededRandom() * total;
     for (let i = 0; i < items.length; i++) {
         random -= weights[i];
         if (random <= 0) return items[i];
@@ -437,7 +443,7 @@ class NaturalTypoEngine {
         
         // Fallback: transpose letters
         if (word.length > 3) {
-            const pos = Math.floor(Math.random() * (word.length - 2)) + 1;
+            const pos = Math.floor(seededRandom() * (word.length - 2)) + 1;
             return word.slice(0, pos) + word[pos + 1] + word[pos] + word.slice(pos + 2);
         }
         
@@ -705,7 +711,7 @@ class OrganicTextGenerator {
     
     generateStreamOfConsciousness() {
         const thoughts = [];
-        const numThoughts = 2 + Math.floor(Math.random() * 4);
+        const numThoughts = 2 + Math.floor(seededRandom() * 4);
         
         for (let i = 0; i < numThoughts; i++) {
             const thought = this.thoughtStream.generateThought(
@@ -778,11 +784,11 @@ class OrganicTextGenerator {
     generateFragmented() {
         const fragments = [];
         const numFragments = this.config.style === 'search' ? 
-            1 + Math.floor(Math.random() * 3) : 
-            2 + Math.floor(Math.random() * 4);
+            1 + Math.floor(seededRandom() * 3) : 
+            2 + Math.floor(seededRandom() * 4);
         
         for (let i = 0; i < numFragments; i++) {
-            const type = Math.random();
+            const type = seededRandom();
             
             if (type < 0.3) {
                 fragments.push(pick(ORGANIC_PATTERNS.fragments.incomplete));
@@ -933,7 +939,7 @@ class OrganicTextGenerator {
         
         // Insert markers naturally
         for (const marker of markers) {
-            const insertPoint = Math.floor(Math.random() * text.length);
+            const insertPoint = Math.floor(seededRandom() * text.length);
             text = text.slice(0, insertPoint) + ' ' + marker + ' ' + text.slice(insertPoint);
         }
         
@@ -958,8 +964,8 @@ class OrganicTextGenerator {
     }
     
     addTimestamp(text) {
-        const hour = Math.floor(Math.random() * 24);
-        const min = Math.floor(Math.random() * 60);
+        const hour = Math.floor(seededRandom() * 24);
+        const min = Math.floor(seededRandom() * 60);
         const timestamp = `[${hour}:${min.toString().padStart(2, '0')}]`;
         
         return timestamp + ' ' + text;
@@ -978,7 +984,7 @@ class OrganicTextGenerator {
     }
     
     breakSentence(sentence) {
-        const breakPoint = Math.floor(sentence.length * (0.3 + Math.random() * 0.4));
+        const breakPoint = Math.floor(sentence.length * (0.3 + seededRandom() * 0.4));
         return sentence.slice(0, breakPoint) + '...';
     }