make-mp-data 3.0.3 → 3.0.5

package/dungeons/text-generation.js

@@ -12,6 +12,77 @@ const days = 92;
 
 /** @typedef  {import("../types.js").Dungeon} Dungeon */
 
+/*
+ * ============================================================================
+ * DATASET OVERVIEW
+ * ============================================================================
+ *
+ * App: Text Generation Demo — showcases DM4's organic text generation
+ * Scale: 8,000 users, ~960K events, 92 days
+ *
+ * A text-heavy SaaS analytics dungeon that exercises every text generation
+ * style available in DM4. Users interact through support tickets, product
+ * reviews, forum posts, search queries, chat messages, social media posts
+ * (Twitter, LinkedIn, Reddit), bug reports, feature requests, onboarding
+ * feedback, charity/wedding comments, and webinar chat.
+ *
+ * Each event type uses a dedicated createTextGenerator() configured with its
+ * own style, tone, formality, keyword banks, typo rates, and authenticity
+ * levels. Text generator styles used: support, review, forum, search,
+ * feedback, chat, email, tweet, comments (9 distinct styles).
+ *
+ * Events:
+ *   social_media_tweet (15) > chat_message (10) > charity_comment_posted (8)
+ *   > wedding_comment_posted (6) > company_announcement_tweet (3)
+ *   > all others (1 each): support ticket, review, forum post, search,
+ *     feedback, email, twitter, linkedin, reddit, bug report, feature
+ *     request, onboarding, tutorial comment, webinar chat, api thread
+ * ============================================================================
+ */
+
+/*
+ * ============================================================================
+ * ANALYTICS HOOKS
+ * ============================================================================
+ *
+ * Hook 1: Power User + Churn Risk Classification
+ *   Type: user
+ *   What: Users with engagement_score > 70 get is_power_user = true.
+ *     Users inactive > 20 days get risk_level = "high_churn", else "healthy".
+ *   Mixpanel report:
+ *     - Insights > user profile breakdown by "is_power_user"
+ *     - Insights > user profile breakdown by "risk_level", cross-reference
+ *       with "user_tier" to see churn risk by plan
+ *
+ * Hook 2: Critical Ticket Auto-Escalation
+ *   Type: event
+ *   What: enterprise_support_ticket events with priority = "critical" get
+ *     escalation_level bumped by 1 (max 3) and auto_escalated = true.
+ *   Mixpanel report:
+ *     - Insights > "enterprise_support_ticket" total events, breakdown by
+ *       auto_escalated
+ *     - Expect: critical tickets show auto_escalated = true
+ *
+ * Hook 3: Critical Bug Flagging
+ *   Type: event
+ *   What: bug_report_submitted events with severity = "critical" AND
+ *     is_reproducible = true get requires_immediate_review = true and a
+ *     random estimated_fix_hours (1-8).
+ *   Mixpanel report:
+ *     - Insights > "bug_report_submitted" total events, breakdown by
+ *       requires_immediate_review
+ *     - Expect: only critical + reproducible bugs are flagged
+ *
+ * Hook 4: Enterprise Satisfaction Survey Injection
+ *   Type: everything
+ *   What: Enterprise-tier users with > 5 events get a
+ *     "satisfaction_survey_triggered" event appended with a 1-10 NPS score.
+ *   Mixpanel report:
+ *     - Insights > "satisfaction_survey_triggered" AVG(score), breakdown by
+ *       product_tier
+ *     - Expect: only enterprise users have survey events
+ * ============================================================================
+ */
 
 // Enterprise support ticket generator with keywords and high authenticity
 const enterpriseSupportGen = createTextGenerator({

package/index.js

@@ -197,13 +197,16 @@ async function main(config) {
 
 		// ! DATA GENERATION ENDS HERE
 
+		// Flush when writeToDisk is enabled OR batch mode activated (to capture tail data)
+		const shouldFlush = validatedConfig.writeToDisk || context.isBatchMode();
+
 		// Step 10: Flush lookup tables to disk (always as CSVs)
-		if (validatedConfig.writeToDisk) {
+		if (shouldFlush) {
 			await flushLookupTablesToDisk(storage, validatedConfig);
 		}
 
-		// Step 11: Flush other storage containers to disk (if writeToDisk enabled)
-		if (validatedConfig.writeToDisk) {
+		// Step 11: Flush other storage containers to disk
+		if (shouldFlush) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
 
@@ -276,7 +279,7 @@ async function generateGroupProfiles(context) {
 		const groupContainer = storage.groupProfilesData[i];
 
 		if (!groupContainer) {
-			console.warn(`Warning: No storage container found for group key: ${groupKey}`);
+			if (config.verbose) console.warn(`Warning: No storage container found for group key: ${groupKey}`);
 			continue;
 		}
 
@@ -319,7 +322,7 @@ async function generateLookupTables(context) {
 		const lookupContainer = storage.lookupTableData[i];
 
 		if (!lookupContainer) {
-			console.warn(`Warning: No storage container found for lookup table: ${key}`);
+			if (config.verbose) console.warn(`Warning: No storage container found for lookup table: ${key}`);
 			continue;
 		}
 
@@ -551,7 +554,6 @@ async function extractFileInfo(storage, config) {
 			}
 		} catch (error) {
 			// If scanning fails, just return empty array
-			console.warn('Warning: Could not scan data directory for files:', error.message);
 		}
 	}
 

package/lib/core/config-validator.js

@@ -11,6 +11,7 @@
 import dayjs from "dayjs";
 import { makeName } from "ak-tools";
 import * as u from "../utils/utils.js";
+import { resolveSoup } from "../templates/soup-presets.js";
 
 /**
  * Infers funnels from the provided events
@@ -106,7 +107,7 @@ export function validateDungeonConfig(config) {
 		token = null,
 		region = "US",
 		writeToDisk = false,
-		verbose = true,
+		verbose = false,
 		soup = {},
 		hook = (record) => record,
 		hasAdSpend = false,
@@ -136,6 +137,12 @@ export function validateDungeonConfig(config) {
 		concurrency = 1;
 	}
 
+	// Auto-enable batch mode for large datasets to prevent OOM
+	if (numEvents >= 2_000_000 && config.batchSize === undefined) {
+		batchSize = 1_000_000;
+		console.warn(`⚠️  Auto-enabling batch mode: numEvents (${numEvents.toLocaleString()}) >= 2M. Using batchSize of ${batchSize.toLocaleString()}.`);
+	}
+
 	// Ensure defaults for deep objects
 	if (!config.superProps) config.superProps = superProps;
 	if (!config.userProps || Object.keys(config?.userProps || {})) config.userProps = userProps;
@@ -148,6 +155,17 @@ export function validateDungeonConfig(config) {
 		throw new Error("Either epochStart or numDays must be provided");
 	}
 
+	// Resolve soup presets (must happen after numDays is computed)
+	const resolved = resolveSoup(soup, numDays);
+	soup = resolved.soup;
+	// Apply suggested birth distribution params if not explicitly set by the dungeon
+	if (resolved.suggestedBornRecentBias !== undefined && config.bornRecentBias === undefined) {
+		config.bornRecentBias = resolved.suggestedBornRecentBias;
+	}
+	if (resolved.suggestedPercentUsersBornInDataset !== undefined && config.percentUsersBornInDataset === undefined) {
+		config.percentUsersBornInDataset = resolved.suggestedPercentUsersBornInDataset;
+	}
+
 	// Use provided name if non-empty string, otherwise generate one
 	if (!name || name === "") {
 		name = makeName();
@@ -166,15 +184,17 @@ export function validateDungeonConfig(config) {
 				throw new Error('Hook string did not evaluate to a function');
 			}
 		} catch (error) {
-			console.warn(`\u26a0\ufe0f Failed to convert hook string to function: ${error.message}`);
-			console.warn('Using default pass-through hook');
+			if (config.verbose !== false) {
+				console.warn(`\u26a0\ufe0f Failed to convert hook string to function: ${error.message}`);
+				console.warn('Using default pass-through hook');
+			}
 			hook = (record) => record;
 		}
 	}
 
 	// Ensure hook is a function
 	if (typeof hook !== 'function') {
-		console.warn('\u26a0\ufe0f Hook is not a function, using default pass-through hook');
+		if (config.verbose !== false) console.warn('\u26a0\ufe0f Hook is not a function, using default pass-through hook');
 		hook = (record) => record;
 	}
 
@@ -337,7 +357,7 @@ function transformSCDPropsWithoutCredentials(config) {
 	}
 
 	// UI job without credentials - convert SCD props to regular props
-	console.log('\u26a0\ufe0f  Service account credentials missing - converting SCD properties to static properties');
+	if (config.verbose !== false) console.log('\u26a0\ufe0f  Service account credentials missing - converting SCD properties to static properties');
 
 	// Ensure userProps and groupProps exist
 	if (!config.userProps) config.userProps = {};
@@ -356,20 +376,20 @@ function transformSCDPropsWithoutCredentials(config) {
 		if (type === "user") {
 			// Add to userProps
 			config.userProps[propKey] = values;
-			console.log(`  \u2713 Converted user SCD property: ${propKey}`);
+			if (config.verbose !== false) console.log(`  \u2713 Converted user SCD property: ${propKey}`);
 		} else {
 			// Add to groupProps for the specific group type
 			if (!config.groupProps[type]) {
 				config.groupProps[type] = {};
 			}
 			config.groupProps[type][propKey] = values;
-			console.log(`  \u2713 Converted group SCD property: ${propKey} (${type})`);
+			if (config.verbose !== false) console.log(`  \u2713 Converted group SCD property: ${propKey} (${type})`);
 		}
 	}
 
 	// Clear out scdProps since we've converted everything
 	config.scdProps = {};
-	console.log('\u2713 SCD properties converted to static properties\n');
+	if (config.verbose !== false) console.log('\u2713 SCD properties converted to static properties\n');
 }
 
 export { inferFunnels, transformSCDPropsWithoutCredentials };

package/lib/core/storage.js

@@ -366,12 +366,12 @@ export class StorageManager {
 		if (config.writeToDisk === false) {
 			const batchSize = config.batchSize || 1_000_000;
 			const numEvents = config.numEvents || 0;
-			
+
 			if (batchSize < numEvents) {
-				throw new Error(
-					`Configuration error: writeToDisk is explicitly set to false but batchSize (${batchSize}) is lower than numEvents (${numEvents}). ` +
-					`This would result in data loss as batched data would be discarded. ` +
-					`Either set writeToDisk to true, increase batchSize to be >= numEvents, or provide a Mixpanel token to send data directly.`
+				console.warn(
+					`⚠️  writeToDisk is false but batchSize (${batchSize.toLocaleString()}) < numEvents (${numEvents.toLocaleString()}). ` +
+					`Batch files will be written to disk temporarily to avoid OOM. ` +
+					`They will be cleaned up after Mixpanel import if a token is provided.`
 				);
 			}
 		}

package/lib/generators/events.js

@@ -52,9 +52,9 @@ export async function makeEvent(
     const chance = u.getChance();
     
     // Extract soup configuration for time distribution
-    // Dynamic peaks: one per week for long ranges, minimum 5
-    const defaultPeaks = Math.max(5, Math.ceil((config.numDays || 30) / 7));
-    const { mean = 0, deviation = 2, peaks = defaultPeaks } = config.soup || {};
+    // Dynamic peaks: enough to flatten DOW interference from chunk boundaries
+    const defaultPeaks = Math.max(5, (config.numDays || 30) * 2);
+    const { mean = 0, deviation = 2, peaks = defaultPeaks, dayOfWeekWeights, hourOfDayWeights } = /** @type {import('../../types').SoupConfig} */ (config.soup) || {};
     
     // Extract feature flags from config
     const {
@@ -102,7 +102,7 @@ export async function makeEvent(
             shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
         } else {
             // TimeSoup returns unix seconds; shift and convert to ISO once
-            const soupTimestamp = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+            const soupTimestamp = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean, dayOfWeekWeights, hourOfDayWeights, context.TIME_SHIFT_SECONDS);
             shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
         }
         // Drop events that would land in the future (Mixpanel rewrites these to "now", causing pile-ups)

package/lib/orchestrators/mixpanel-sender.js

@@ -6,7 +6,6 @@
 /** @typedef {import('../../types').Context} Context */
 
 import dayjs from "dayjs";
-import path from "path";
 import { comma, ls, rm } from "ak-tools";
 import * as u from "../utils/utils.js";
 import mp from "mixpanel-import";
@@ -22,8 +21,6 @@ export async function sendToMixpanel(context) {
 		adSpendData,
 		eventData,
 		groupProfilesData,
-		lookupTableData,
-		mirrorEventData,
 		scdTableData,
 		userProfilesData,
 		groupEventData
@@ -41,7 +38,7 @@ export async function sendToMixpanel(context) {
 
 	const importResults = { events: {}, users: {}, groups: [] };
 	const isBATCH_MODE = context.isBatchMode();
-	const NODE_ENV = process.env.NODE_ENV || "unknown";
+	_verbose = config.verbose !== false;
 
 	/** @type {import('mixpanel-import').Creds} */
 	const creds = { token };
@@ -236,15 +233,20 @@ export async function sendToMixpanel(context) {
 	if (!writeToDisk && isBATCH_MODE) {
 		const writeDir = eventData?.getWriteDir?.() || userProfilesData?.getWriteDir?.();
 		if (writeDir) {
+			const configName = context.config.name;
 			const listDir = await ls(writeDir);
 			// @ts-ignore
-			const files = listDir.filter(f =>
-				f.includes('-EVENTS') ||
-				f.includes('-USERS') ||
-				f.includes('-ADSPEND') ||
-				f.includes('-GROUPS') ||
-				f.includes('-GROUP-EVENTS')
-			);
+			const files = listDir.filter(f => {
+				if (configName && !f.includes(configName)) return false;
+				return f.includes('-EVENTS') ||
+					f.includes('-USERS') ||
+					f.includes('-ADSPEND') ||
+					f.includes('-GROUPS') ||
+					f.includes('-GROUP-EVENTS') ||
+					f.includes('-SCD') ||
+					f.includes('-MIRROR') ||
+					f.includes('-LOOKUP');
+			});
 			for (const file of files) {
 				await rm(file);
 			}
@@ -255,9 +257,10 @@ export async function sendToMixpanel(context) {
 }
 
 /**
- * Simple logging function
+ * Logging function that respects verbose config
  * @param {string} message - Message to log
  */
+let _verbose = true;
 function log(message) {
-	console.log(message);
+	if (_verbose) console.log(message);
 }

package/lib/orchestrators/user-loop.js

@@ -59,6 +59,7 @@ export async function userLoop(context) {
 	let cancelled = false;
 	const onSigint = () => {
 		cancelled = true;
+		USER_CONN.clearQueue();
 		if (verbose) console.log(`\n\nStopping generation (Ctrl+C)...\n`);
 	};
 	process.on('SIGINT', onSigint);
@@ -109,7 +110,7 @@ export async function userLoop(context) {
 			if (userIsBornInDataset) {
 				let biasedCreated = dayjs(created).subtract(daysShift, 'd');
 
-				if (bornRecentBias > 0) {
+				if (bornRecentBias !== 0) {
 					// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
 					const datasetStart = dayjs.unix(global.FIXED_BEGIN);
 					const datasetEnd = dayjs.unix(context.FIXED_NOW);
@@ -117,10 +118,17 @@ export async function userLoop(context) {
 					// Clamp userPosition to [0, 1] to handle edge cases from rounding in time calculations
 					const userPosition = Math.max(0, Math.min(1, biasedCreated.diff(datasetStart) / totalDuration));
 
-					// Apply power function to bias toward recent (higher values)
-					// exponent < 1 shifts distribution toward 1 (recent)
-					const exponent = 1 - (bornRecentBias * 0.7); // 0.3 bias -> 0.79 exponent (gentle nudge)
-					const biasedPosition = Math.pow(userPosition, exponent);
+					let biasedPosition;
+					if (bornRecentBias > 0) {
+						// Positive bias: exponent < 1 shifts distribution toward 1 (recent)
+						const exponent = 1 - (bornRecentBias * 0.7); // 0.3 bias -> 0.79 exponent (gentle nudge)
+						biasedPosition = Math.pow(userPosition, exponent);
+					} else {
+						// Negative bias: mirror the power function to shift toward 0 (early)
+						// -0.3 bias -> 0.79 exponent applied to (1 - position), then mirrored back
+						const exponent = 1 - (Math.abs(bornRecentBias) * 0.7);
+						biasedPosition = 1 - Math.pow(1 - userPosition, exponent);
+					}
 
 					// Convert back to timestamp
 					biasedCreated = datasetStart.add(biasedPosition * totalDuration, 'millisecond');
@@ -233,7 +241,7 @@ export async function userLoop(context) {
 
 			// ALL SUBSEQUENT EVENTS (funnels for converted users, standalone for all)
 			let userChurned = false;
-			while (numEventsPreformed < numEventsThisUserWillPreform) {
+			while (numEventsPreformed < numEventsThisUserWillPreform && !cancelled) {
 				let newEvents;
 				if (usageFunnels.length && userConverted) {
 					const currentFunnel = chance.pickone(usageFunnels);

package/lib/templates/soup-presets.js

@@ -0,0 +1,188 @@
+/**
+ * TimeSoup preset configurations
+ * Each preset defines time distribution parameters that produce distinct patterns.
+ *
+ * Parameters:
+ * - peaks(numDays): function returning number of Gaussian clusters
+ * - deviation: controls peak width (higher = tighter)
+ * - mean: offset from chunk center (0 = centered)
+ * - dayOfWeekWeights: 7-element array [Sun..Sat], max=1.0, null to disable
+ * - hourOfDayWeights: 24-element array [0h..23h UTC], max=1.0, null to disable
+ *
+ * Some presets also suggest bornRecentBias and percentUsersBornInDataset,
+ * but those are top-level dungeon config — presets only set them if not already specified.
+ */
+
+// Real-world Mixpanel DOW pattern: weekday-heavy, Saturday valley
+export const REAL_DOW = [0.637, 1.0, 0.999, 0.998, 0.966, 0.802, 0.528];
+
+// Real-world Mixpanel HOD pattern: early-morning peak (UTC), afternoon valley
+export const REAL_HOD = [
+	0.949, 0.992, 0.998, 0.946, 0.895, 0.938, 1.0, 0.997,
+	0.938, 0.894, 0.827, 0.786, 0.726, 0.699, 0.688, 0.643,
+	0.584, 0.574, 0.554, 0.576, 0.604, 0.655, 0.722, 0.816
+];
+
+// Flat weights (no cyclical pattern)
+export const FLAT_DOW = [1, 1, 1, 1, 1, 1, 1];
+export const FLAT_HOD = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
+
+/** @type {Record<string, {peaks: (numDays: number) => number, deviation: number, mean: number, dayOfWeekWeights: number[]|null, hourOfDayWeights: number[]|null, bornRecentBias?: number, percentUsersBornInDataset?: number}>} */
+export const SOUP_PRESETS = {
+	/**
+	 * steady — Mature SaaS / Stable Product
+	 * Nearly flat day-over-day, slight weekly pattern, minimal growth trend.
+	 */
+	steady: {
+		peaks: (numDays) => Math.max(5, numDays * 2),
+		deviation: 1.5,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0.1,
+		percentUsersBornInDataset: 10,
+	},
+
+	/**
+	 * growth — Growing Startup (DEFAULT)
+	 * Gradual uptrend with visible weekly peaks. This is the default behavior.
+	 */
+	growth: {
+		peaks: (numDays) => Math.max(5, numDays * 2),
+		deviation: 2,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0.3,
+		percentUsersBornInDataset: 15,
+	},
+
+	/**
+	 * spiky — Event-Driven / Bursty
+	 * Clear peaks and valleys, dramatic variation. Fewer Gaussian clusters + tight deviation.
+	 */
+	spiky: {
+		peaks: (numDays) => Math.max(5, Math.ceil(numDays / 10)),
+		deviation: 3.5,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0.3,
+		percentUsersBornInDataset: 20,
+	},
+
+	/**
+	 * seasonal — Strong Cyclical Patterns
+	 * 3-4 major waves across the dataset. Very few peaks create dramatic macro trends.
+	 */
+	seasonal: {
+		peaks: () => 4,
+		deviation: 2.5,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0.2,
+		percentUsersBornInDataset: 25,
+	},
+
+	/**
+	 * global — Distributed Users Across Timezones
+	 * Very flat hourly + daily distribution. No cyclical patterns.
+	 */
+	global: {
+		peaks: (numDays) => Math.max(5, numDays * 2),
+		deviation: 1,
+		mean: 0,
+		dayOfWeekWeights: FLAT_DOW,
+		hourOfDayWeights: FLAT_HOD,
+		bornRecentBias: 0,
+		percentUsersBornInDataset: 10,
+	},
+
+	/**
+	 * churny — High Churn / Declining Product
+	 * Flat distribution (no growth trend). All users pre-exist the dataset,
+	 * so there's no acceleration. Combine with an "everything" hook that
+	 * filters late events to create a true declining shape.
+	 */
+	churny: {
+		peaks: (numDays) => Math.max(5, numDays * 2),
+		deviation: 2,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0,
+		percentUsersBornInDataset: 5,
+	},
+
+	/**
+	 * chaotic — Unpredictable / Irregular Patterns
+	 * Few peaks + very tight clustering = dramatic bursts separated by quiet stretches.
+	 */
+	chaotic: {
+		peaks: (numDays) => Math.max(3, Math.ceil(numDays / 20)),
+		deviation: 4,
+		mean: 0,
+		dayOfWeekWeights: REAL_DOW,
+		hourOfDayWeights: REAL_HOD,
+		bornRecentBias: 0.5,
+		percentUsersBornInDataset: 40,
+	},
+};
+
+/** @type {string[]} */
+export const PRESET_NAMES = Object.keys(SOUP_PRESETS);
+
+/**
+ * Resolves a soup config — handles string presets, preset+overrides, and raw objects.
+ * @param {string | object} soup - Soup config from dungeon
+ * @param {number} numDays - Number of days in the dataset
+ * @returns {{ soup: object, suggestedBornRecentBias?: number, suggestedPercentUsersBornInDataset?: number }}
+ */
+export function resolveSoup(soup, numDays) {
+	if (!soup) return { soup: {} };
+
+	// String preset: "growth", "spiky", etc.
+	if (typeof soup === 'string') {
+		const preset = SOUP_PRESETS[soup];
+		if (!preset) {
+			throw new Error(`Unknown soup preset: "${soup}". Valid presets: ${PRESET_NAMES.join(', ')}`);
+		}
+		return {
+			soup: {
+				peaks: preset.peaks(numDays),
+				deviation: preset.deviation,
+				mean: preset.mean,
+				dayOfWeekWeights: preset.dayOfWeekWeights,
+				hourOfDayWeights: preset.hourOfDayWeights,
+			},
+			suggestedBornRecentBias: preset.bornRecentBias,
+			suggestedPercentUsersBornInDataset: preset.percentUsersBornInDataset,
+		};
+	}
+
+	// Object with preset key: { preset: "growth", deviation: 3 }
+	if (typeof soup === 'object' && soup.preset) {
+		const preset = SOUP_PRESETS[soup.preset];
+		if (!preset) {
+			throw new Error(`Unknown soup preset: "${soup.preset}". Valid presets: ${PRESET_NAMES.join(', ')}`);
+		}
+		const base = {
+			peaks: preset.peaks(numDays),
+			deviation: preset.deviation,
+			mean: preset.mean,
+			dayOfWeekWeights: preset.dayOfWeekWeights,
+			hourOfDayWeights: preset.hourOfDayWeights,
+		};
+		// Apply overrides (excluding the 'preset' key itself)
+		const { preset: _, ...overrides } = soup;
+		return {
+			soup: { ...base, ...overrides },
+			suggestedBornRecentBias: preset.bornRecentBias,
+			suggestedPercentUsersBornInDataset: preset.percentUsersBornInDataset,
+		};
+	}
+
+	// Raw object: { peaks: 10, deviation: 2 } — pass through unchanged
+	return { soup };
+}

package/lib/utils/utils.js

@@ -1180,7 +1180,17 @@ let soupHits = 0;
  * Divides the range into `peaks` chunks, picks one randomly, then samples within it.
  * Returns unix seconds (not ISO string) for performance — caller converts once.
  */
-function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0) {
+// Default day-of-week weights (0=Sun, 1=Mon, ..., 6=Sat) — derived from real Mixpanel data
+const DEFAULT_DOW_WEIGHTS = [0.637, 1.0, 0.999, 0.998, 0.966, 0.802, 0.528];
+
+// Default hour-of-day weights (0=midnight, ..., 23=11pm UTC) — derived from real Mixpanel data
+const DEFAULT_HOD_WEIGHTS = [
+	0.949, 0.992, 0.998, 0.946, 0.895, 0.938, 1.0, 0.997,
+	0.938, 0.894, 0.827, 0.786, 0.726, 0.699, 0.688, 0.643,
+	0.584, 0.574, 0.554, 0.576, 0.604, 0.655, 0.722, 0.816
+];
+
+function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS, timeShiftSeconds = 0) {
 	if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix();
 	if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
 	const chance = getChance();
@@ -1193,21 +1203,57 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	}
 	const chunkSize = totalRange / peaks;
 
-	// Select a random chunk based on the number of peaks
+	// Phase 1: Gaussian chunk sampling (macro trend across the time range)
 	const peakIndex = integer(0, peaks - 1);
 	const chunkStart = earliestTime + peakIndex * chunkSize;
 	const chunkEnd = chunkStart + chunkSize;
 	const chunkMid = (chunkStart + chunkEnd) / 2;
-
-	// Generate offset from normal distribution, clamp to chunk boundaries
 	const maxDeviation = chunkSize / deviation;
 	const offset = chance.normal({ mean: mean, dev: maxDeviation });
 	const proposedTime = chunkMid + offset;
 	const clampedTime = Math.max(chunkStart, Math.min(chunkEnd, proposedTime));
-	const finalTime = Math.max(earliestTime, Math.min(latestTime, clampedTime));
+	let candidate = Math.max(earliestTime, Math.min(latestTime, clampedTime));
+
+	// Phase 2: DOW accept/reject — retry if day-of-week doesn't pass weight check
+	if (dayOfWeekWeights) {
+		for (let attempt = 0; attempt < 50; attempt++) {
+			const dow = new Date((candidate + timeShiftSeconds) * 1000).getUTCDay();
+			if (chance.random() < dayOfWeekWeights[dow]) break;
+			// Rejected — resample from Gaussian chunks
+			const pi = integer(0, peaks - 1);
+			const cs = earliestTime + pi * chunkSize;
+			const ce = cs + chunkSize;
+			const cm = (cs + ce) / 2;
+			const md = chunkSize / deviation;
+			const off = chance.normal({ mean: mean, dev: md });
+			const pt = cm + off;
+			candidate = Math.max(earliestTime, Math.min(latestTime, Math.max(cs, Math.min(ce, pt))));
+		}
+	}
+
+	// Phase 3: Redistribute hour-of-day (changes only hour within same day)
+	if (hourOfDayWeights) {
+		const shifted = candidate + timeShiftSeconds;
+		const d = new Date(shifted * 1000);
+		const currentMinute = d.getUTCMinutes();
+		const currentSecond = d.getUTCSeconds();
+
+		const totalHodWeight = hourOfDayWeights.reduce((s, w) => s + w, 0);
+		let roll = chance.random() * totalHodWeight;
+		let newHour = 0;
+		for (let h = 0; h < 24; h++) {
+			roll -= hourOfDayWeights[h];
+			if (roll <= 0) { newHour = h; break; }
+		}
+
+		const dayStartShifted = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) / 1000;
+		const newShifted = dayStartShifted + newHour * 3600 + currentMinute * 60 + currentSecond;
+		candidate = newShifted - timeShiftSeconds;
+		candidate = Math.max(earliestTime, Math.min(latestTime, candidate));
+	}
 
 	soupHits++;
-	return finalTime;
+	return candidate;
 }
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "3.0.3",
+	"version": "3.0.5",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",