@ak--47/dungeon-master 1.4.3 → 1.4.5

package/index.js

@@ -157,31 +157,49 @@ async function runDungeon(config) {
 
 		// Step 4: Generate ad spend data (if enabled)
 		if (validatedConfig.hasAdSpend) {
+			context.reportProgress({ phase: "step", step: "adspend", status: "start" });
+			const _t4 = Date.now();
 			await generateAdSpendData(context);
+			context.reportProgress({ phase: "step", step: "adspend", status: "complete", duration: Date.now() - _t4 });
 		}
 
 		if (context.config.verbose) logger.info('Starting user and event generation...');
 		// Step 5: Main user and event generation
+		context.reportProgress({ phase: "step", step: "users", status: "start" });
+		const _t5 = Date.now();
 		await userLoop(context);
+		context.reportProgress({ phase: "step", step: "users", status: "complete", duration: Date.now() - _t5 });
 
 		// Step 6: Generate group profiles (if configured)
 		if (validatedConfig.groupKeys && validatedConfig.groupKeys.length > 0) {
+			context.reportProgress({ phase: "step", step: "group-profiles", status: "start" });
+			const _t6 = Date.now();
 			await generateGroupProfiles(context);
+			context.reportProgress({ phase: "step", step: "group-profiles", status: "complete", duration: Date.now() - _t6 });
 		}
 
 		// Step 7: Generate group SCDs (if configured)
 		if (validatedConfig.scdProps && validatedConfig.groupKeys && validatedConfig.groupKeys.length > 0) {
+			context.reportProgress({ phase: "step", step: "group-scds", status: "start" });
+			const _t7 = Date.now();
 			await generateGroupSCDs(context);
+			context.reportProgress({ phase: "step", step: "group-scds", status: "complete", duration: Date.now() - _t7 });
 		}
 
 		// Step 8: Generate lookup tables (if configured)
 		if (validatedConfig.lookupTables && validatedConfig.lookupTables.length > 0) {
+			context.reportProgress({ phase: "step", step: "lookups", status: "start" });
+			const _t8 = Date.now();
 			await generateLookupTables(context);
+			context.reportProgress({ phase: "step", step: "lookups", status: "complete", duration: Date.now() - _t8 });
 		}
 
 		// Step 9: Generate mirror datasets (if configured)
 		if (validatedConfig.mirrorProps && Object.keys(validatedConfig.mirrorProps).length > 0) {
+			context.reportProgress({ phase: "step", step: "mirrors", status: "start" });
+			const _t9 = Date.now();
 			await makeMirror(context);
+			context.reportProgress({ phase: "step", step: "mirrors", status: "complete", duration: Date.now() - _t9 });
 		}
 
 		if (context.config.verbose) logger.info('Data generation completed successfully');
@@ -191,21 +209,23 @@ async function runDungeon(config) {
 		// Flush when writeToDisk is enabled OR batch mode activated (to capture tail data)
 		const shouldFlush = validatedConfig.writeToDisk || context.isBatchMode();
 
-		// Step 10: Flush lookup tables to disk (always as CSVs)
+		// Step 10-11: Flush to disk
 		if (shouldFlush) {
+			context.reportProgress({ phase: "step", step: "flush", status: "start" });
+			const _tFlush = Date.now();
 			await flushLookupTablesToDisk(storage, validatedConfig);
-		}
-
-		// Step 11: Flush other storage containers to disk
-		if (shouldFlush) {
 			await flushStorageToDisk(storage, validatedConfig);
+			context.reportProgress({ phase: "step", step: "flush", status: "complete", duration: Date.now() - _tFlush });
 		}
 
 		// Step 12: Send to Mixpanel (if token provided)
 		// Now happens AFTER disk flush so batch files are available for import
 		let importResults;
 		if (validatedConfig.token) {
+			context.reportProgress({ phase: "step", step: "import", status: "start" });
+			const _t12 = Date.now();
 			importResults = await sendToMixpanel(context);
+			context.reportProgress({ phase: "step", step: "import", status: "complete", duration: Date.now() - _t12 });
 		}
 
 		// Step 13: Compile results
@@ -214,6 +234,8 @@ async function runDungeon(config) {
 
 		const extractedData = extractStorageData(storage);
 
+		const progressSummary = context.getProgressSummary();
+
 		return {
 			...extractedData,
 			importResults,
@@ -221,7 +243,8 @@ async function runDungeon(config) {
 			time: { start, end, delta, human },
 			operations: context.getOperations(),
 			eventCount: context.getStoredEventCount(),
-			userCount: context.getUserCount()
+			userCount: context.getUserCount(),
+			...(progressSummary.updates > 0 || progressSummary.errors > 0 ? { progress: progressSummary } : {})
 		};
 
 	} catch (error) {

package/lib/core/config-validator.js

@@ -283,7 +283,7 @@ export function validateDungeonConfig(config) {
 		hasAnonIds = false,
 		hasSessionIds = false,
 		sessionTimeout = 30,
-		format = "csv",
+		format,
 		token = null,
 		region = "US",
 		writeToDisk = false,
@@ -592,7 +592,7 @@ export function validateDungeonConfig(config) {
 		avgDevicePerUser,
 		hasSessionIds,
 		sessionTimeout: (typeof sessionTimeout === 'number' && sessionTimeout > 0) ? sessionTimeout : 30,
-		format,
+		format: format || (typeof writeToDisk === 'string' && writeToDisk.startsWith('gs://') ? 'json' : 'csv'),
 		token,
 		region,
 		writeToDisk,

package/lib/core/context.js

@@ -8,6 +8,8 @@
 /** @typedef {import('../../types.js').Context} Context */
 /** @typedef {import('../../types.js').RuntimeState} RuntimeState */
 /** @typedef {import('../../types.js').Defaults} Defaults */
+/** @typedef {import('../../types.js').ProgressUpdate} ProgressUpdate */
+/** @typedef {import('../../types.js').ProgressSummary} ProgressSummary */
 
 import dayjs from "dayjs";
 import { campaigns, devices, locations } from '../templates/defaults.js';
@@ -78,6 +80,58 @@ function createRuntimeState() {
 	};
 }
 
+/**
+ * @param {Dungeon} config
+ * @returns {{ reportProgress: (update: ProgressUpdate) => void, getProgressSummary: () => ProgressSummary }}
+ */
+function createProgressReporter(config) {
+	const interval = config.progressInterval ?? 500;
+	const verbose = config.verbose || false;
+	let callback = config.onProgress ?? null;
+	let lastFireTime = 0;
+	let errorCount = 0;
+	let totalUpdates = 0;
+	let disabled = false;
+
+	if (callback !== null && typeof callback !== 'function') {
+		if (verbose) console.warn(`[dungeon-master] onProgress is not a function (got ${typeof callback}), ignoring`);
+		callback = null;
+	}
+
+	function reportProgress(/** @type {ProgressUpdate} */ update) {
+		if (!callback || disabled) return;
+
+		const isThrottled = update.phase === 'generation' || update.phase === 'import';
+		if (isThrottled) {
+			const now = Date.now();
+			if (now - lastFireTime < interval) return;
+			lastFireTime = now;
+		}
+
+		try {
+			const result = /** @type {any} */ (callback(update));
+			totalUpdates++;
+			if (result && typeof result.then === 'function') {
+				result.then(undefined, (/** @type {any} */ err) => {
+					errorCount++;
+					if (verbose) console.warn(`[dungeon-master] onProgress async error (${errorCount}/3): ${err?.message || err}`);
+					if (errorCount >= 3) disabled = true;
+				});
+			}
+		} catch (err) {
+			errorCount++;
+			if (verbose) console.warn(`[dungeon-master] onProgress error (${errorCount}/3): ${err?.message || err}`);
+			if (errorCount >= 3) disabled = true;
+		}
+	}
+
+	function getProgressSummary() {
+		return { updates: totalUpdates, errors: errorCount, disabled };
+	}
+
+	return { reportProgress, getProgressSummary };
+}
+
 /**
  * Context factory that creates a complete context object for data generation
  * @param {Dungeon} config - Validated configuration object
@@ -99,12 +153,16 @@ export function createContext(config, storage = null, timeConstants = {}) {
 	runtime.verbose = config.verbose || false;
 	runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
 
+	const { reportProgress, getProgressSummary } = createProgressReporter(config);
+
 	const context = {
 		config,
 		storage,
 		defaults,
 		campaigns: campaignData,
 		runtime,
+		reportProgress,
+		getProgressSummary,
 
 		// Helper methods for updating state
 		incrementOperations() {

package/lib/orchestrators/mixpanel-sender.js

@@ -63,6 +63,21 @@ export async function sendToMixpanel(context) {
 		workers: 35
 	};
 
+	const hasProgressCb = typeof config.onProgress === 'function';
+	function makeProgressCallback(total) {
+		if (!hasProgressCb) return undefined;
+		return (recordType, processed, requests, eps, bytesProcessed) => {
+			context.reportProgress({
+				phase: "import",
+				recordType,
+				processed,
+				total,
+				eps,
+				bytesProcessed
+			});
+		};
+	}
+
 	log(`\n${'─'.repeat(50)}`);
 	log(`  Importing data to Mixpanel (${region})`);
 	log(`${'─'.repeat(50)}\n`);
@@ -76,9 +91,11 @@ export async function sendToMixpanel(context) {
 			const files = eventData.getWrittenFiles();
 			if (files.length > 0) eventDataToImport = files;
 		}
+		const eventTotal = Array.isArray(eventDataToImport) ? eventDataToImport.length : 0;
 		const imported = await mp(creds, eventDataToImport, {
 			recordType: "event",
 			...commonOpts,
+			progressCallback: makeProgressCallback(eventTotal),
 		});
 		log(`  -> ${comma(imported.success)} events sent\n`);
 		importResults.events = imported;
@@ -93,9 +110,11 @@ export async function sendToMixpanel(context) {
 			const files = userProfilesData.getWrittenFiles();
 			if (files.length > 0) userProfilesToImport = files;
 		}
+		const userTotal = Array.isArray(userProfilesToImport) ? userProfilesToImport.length : 0;
 		const imported = await mp(creds, userProfilesToImport, {
 			recordType: "user",
 			...commonOpts,
+			progressCallback: makeProgressCallback(userTotal),
 		});
 		log(`  -> ${comma(imported.success)} user profiles sent\n`);
 		importResults.users = imported;
@@ -110,9 +129,11 @@ export async function sendToMixpanel(context) {
 			const files = adSpendData.getWrittenFiles();
 			if (files.length > 0) adSpendDataToImport = files;
 		}
+		const adTotal = Array.isArray(adSpendDataToImport) ? adSpendDataToImport.length : 0;
 		const imported = await mp(creds, adSpendDataToImport, {
 			recordType: "event",
 			...commonOpts,
+			progressCallback: makeProgressCallback(adTotal),
 		});
 		log(`  -> ${comma(imported.success)} ad spend events sent\n`);
 		importResults.adSpend = imported;
@@ -121,7 +142,8 @@ export async function sendToMixpanel(context) {
 	// Import group profiles
 	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
 		for (const groupEntity of groupProfilesData) {
-			if (!groupEntity || groupEntity.length === 0) continue;
+			if (!groupEntity) continue;
+		if (groupEntity.length === 0 && !isBATCH_MODE) continue;
 			const groupKey = groupEntity?.groupKey;
 			log(`  Group Profiles (${groupKey})`);
 			let groupProfilesToImport = u.deepClone(groupEntity);
@@ -130,10 +152,12 @@ export async function sendToMixpanel(context) {
 				const files = groupEntity.getWrittenFiles();
 				if (files.length > 0) groupProfilesToImport = files;
 			}
+			const groupTotal = Array.isArray(groupProfilesToImport) ? groupProfilesToImport.length : 0;
 			const imported = await mp({ token, groupKey }, groupProfilesToImport, {
 				recordType: "group",
 				...commonOpts,
 				groupKey,
+				progressCallback: makeProgressCallback(groupTotal),
 			});
 			log(`  -> ${comma(imported.success)} ${groupKey} profiles sent\n`);
 			importResults.groups.push(imported);
@@ -141,7 +165,7 @@ export async function sendToMixpanel(context) {
 	}
 
 	// Import group events
-	if (groupEventData?.length > 0) {
+	if (groupEventData?.length > 0 || (isBATCH_MODE && groupEventData)) {
 		log(`  Group Events`);
 		let groupEventDataToImport = u.deepClone(groupEventData);
 		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEventData.length === 0);
@@ -149,9 +173,11 @@ export async function sendToMixpanel(context) {
 			const files = groupEventData.getWrittenFiles();
 			if (files.length > 0) groupEventDataToImport = files;
 		}
+		const groupEvTotal = Array.isArray(groupEventDataToImport) ? groupEventDataToImport.length : 0;
 		const imported = await mp(creds, groupEventDataToImport, {
 			recordType: "event",
 			...commonOpts,
+			progressCallback: makeProgressCallback(groupEvTotal),
 		});
 		log(`  -> ${comma(imported.success)} group events sent\n`);
 		importResults.groupEvents = imported;
@@ -196,6 +222,7 @@ export async function sendToMixpanel(context) {
 				}
 
 				try {
+					const scdTotal = Array.isArray(scdDataToImport) ? scdDataToImport.length : 0;
 					const imported = await mp(
 						{
 							token,
@@ -204,7 +231,7 @@ export async function sendToMixpanel(context) {
 							project: projectId
 						},
 						scdDataToImport,
-						options
+						{ ...options, progressCallback: makeProgressCallback(scdTotal) }
 					);
 					log(`  -> ${comma(imported.success)} ${scdKey} SCD entries sent\n`);
 					importResults[`${scdKey}_scd`] = imported;

package/lib/orchestrators/user-loop.js

@@ -104,6 +104,16 @@ export async function userLoop(context) {
 				]);
 			}
 
+			context.reportProgress({
+				phase: "generation",
+				users: context.getUserCount(),
+				events: context.getEventCount(),
+				eps,
+				memory: memUsed,
+				elapsed: duration,
+				percentComplete: Math.min(100, Math.round((context.getUserCount() / numUsers) * 100))
+			});
+
 			const userId = chance.guid();
 			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW, avgDevicePerUser });
 			const { distinct_id, created } = user;

package/lib/utils/utils.js

@@ -579,7 +579,16 @@ function streamJSON(filePath, data, options = {}) {
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
-			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+			const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				writeStream = gzipStream;
+			} else {
+				writeStream = gcsStream;
+			}
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
@@ -593,9 +602,9 @@ function streamJSON(filePath, data, options = {}) {
 			writeStream.write(JSON.stringify(item) + '\n');
 		});
 		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(filePath);
-		});
+		if (!filePath?.startsWith('gs://')) {
+			writeStream.on('finish', () => resolve(filePath));
+		}
 		writeStream.on('error', reject);
 	});
 }
@@ -607,7 +616,16 @@ function streamCSV(filePath, data, options = {}) {
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
-			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+			const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				writeStream = gzipStream;
+			} else {
+				writeStream = gcsStream;
+			}
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
@@ -635,9 +653,9 @@ function streamCSV(filePath, data, options = {}) {
 		});
 
 		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(filePath);
-		});
+		if (!filePath?.startsWith('gs://')) {
+			writeStream.on('finish', () => resolve(filePath));
+		}
 		writeStream.on('error', reject);
 	});
 }
@@ -710,20 +728,23 @@ async function streamParquet(filePath, data, options = {}) {
 	});
 
 	if (filePath?.startsWith('gs://')) {
-		// For GCS, write to buffer first, then upload
 		// @ts-ignore
 		const arrayBuffer = parquetWriteBuffer({ columnData });
 		const { bucket, file } = parseGCSUri(filePath);
-
-		const writeStream = storage.bucket(bucket).file(file).createWriteStream({
-			gzip: gzip || true // Always gzip for GCS
-		});
+		const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
 
 		return new Promise((resolve, reject) => {
-			writeStream.write(Buffer.from(arrayBuffer));
-			writeStream.end();
-			writeStream.on('finish', () => resolve(filePath));
-			writeStream.on('error', reject);
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				gzipStream.write(Buffer.from(arrayBuffer));
+				gzipStream.end();
+			} else {
+				gcsStream.write(Buffer.from(arrayBuffer));
+				gcsStream.end();
+			}
 		});
 	} else {
 		// For local files

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ak--47/dungeon-master",
-	"version": "1.4.3",
+	"version": "1.4.5",
 	"description": "generate fancy datasets",
 	"type": "module",
 	"main": "index.js",
@@ -26,7 +26,8 @@
 		"scripts/",
 		"package.json",
 		"README.md",
-		"CHANGELOG.md"
+		"CHANGELOG.md",
+		"HOOKS.md"
 	],
 	"engines": {
 		"node": ">=18.0.0"

package/types.d.ts

@@ -119,6 +119,24 @@ export interface Dungeon {
     gzip?: boolean;
     /** If true, prints progress to stdout during generation. */
     verbose?: boolean;
+    /**
+     * Optional callback that receives periodic progress updates during generation,
+     * import, and pipeline step transitions. Fire-and-forget: the callback is never
+     * awaited. If it throws 3 times, it is silently disabled for the rest of the job.
+     *
+     * The `update` argument is a discriminated union on `phase`:
+     * - `"generation"` — user/event counts, EPS, memory, percent complete
+     * - `"import"` — record type, processed/total counts, EPS, bytes
+     * - `"step"` — pipeline step name with start/complete status and duration
+     *
+     * @example
+     * onProgress: (update) => {
+     *   if (update.phase === 'generation') ws.send(JSON.stringify(update));
+     * }
+     */
+    onProgress?: (update: ProgressUpdate) => void;
+    /** Minimum interval (ms) between progress callback invocations. Default: 500. Only throttles `generation` and `import` phases; `step` updates always fire immediately. */
+    progressInterval?: number;
     /**
      * @deprecated Prefer `avgDevicePerUser`. `true` is now an alias for `avgDevicePerUser: 1`
      * (single sticky device per user, every event stamped with that `device_id`). `false`
@@ -657,6 +675,12 @@ export interface Context {
     incrementUserCount(): void;
     incrementEventCount(): void;
     isBatchMode(): boolean;
+
+    // Progress callback
+    /** Fire a progress update to the caller's `onProgress` callback (throttled, fault-tolerant). */
+    reportProgress(update: ProgressUpdate): void;
+    /** Return the progress callback summary (updates delivered, errors, disabled flag). */
+    getProgressSummary(): ProgressSummary;
 }
 
 /**
@@ -1053,8 +1077,61 @@ export type Result = {
     userCount?: number;
     groupCount?: number;
     avgEPS?: number;
+    /** Progress callback summary. Only present when `onProgress` was provided. */
+    progress?: ProgressSummary;
 };
 
+// ============= Progress Callback Types =============
+
+/** Discriminator for progress update types. */
+export type ProgressPhase = "generation" | "import" | "step";
+
+/** Progress update emitted during user/event generation (throttled to `progressInterval`). */
+export interface ProgressGeneration {
+    phase: "generation";
+    users: number;
+    events: number;
+    eps: number;
+    memory: string;
+    elapsed: string;
+    percentComplete: number;
+}
+
+/** Progress update emitted during Mixpanel import (throttled to `progressInterval`). */
+export interface ProgressImport {
+    phase: "import";
+    recordType: string;
+    processed: number;
+    total: number;
+    eps: string;
+    bytesProcessed: number;
+}
+
+/** Progress update emitted at pipeline step boundaries (not throttled). */
+export interface ProgressStep {
+    phase: "step";
+    step: string;
+    status: "start" | "complete";
+    /** Milliseconds elapsed (only present on `status: "complete"`). */
+    duration?: number;
+}
+
+/** Discriminated union of all progress update types. Discriminate on the `phase` field. */
+export type ProgressUpdate = ProgressGeneration | ProgressImport | ProgressStep;
+
+/** Convenience type for the `onProgress` callback signature. */
+export type ProgressCallback = (update: ProgressUpdate) => void;
+
+/** Summary of progress callback activity, included in the job result. */
+export interface ProgressSummary {
+    /** Total number of updates successfully delivered to the callback. */
+    updates: number;
+    /** Number of times the callback threw (0-3; disabled after 3). */
+    errors: number;
+    /** True if the callback was disabled due to repeated failures. */
+    disabled: boolean;
+}
+
 // ============= Advanced Feature Types =============
 
 /**