make-mp-data 2.0.22 → 2.0.23

package/entry.js

@@ -27,6 +27,7 @@ import getCliParams from './lib/cli/cli.js';
             const simpleConfig = await import('./dungeons/simple.js');
             finalConfig = { ...simpleConfig.default, ...cliConfig };
         }
+
 		
 		
         const result = await main(finalConfig);        

package/index.js

@@ -194,8 +194,10 @@ async function main(config) {
 			await generateCharts(context);
 		}
 
-		// Step 11a: Always flush lookup tables to disk (regardless of writeToDisk setting)
-		await flushLookupTablesToDisk(storage, validatedConfig);
+		// Step 11a:  flush lookup tables to disk (always as CSVs)
+		if (validatedConfig.writeToDisk) {
+			await flushLookupTablesToDisk(storage, validatedConfig);
+		}
 
 		// Step 11b: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
@@ -217,7 +219,7 @@ async function main(config) {
 		return {
 			...extractedData,
 			importResults,
-			files: extractFileInfo(storage),
+			files: await extractFileInfo(storage, validatedConfig),
 			time: { start, end, delta, human },
 			operations: context.getOperations(),
 			eventCount: context.getEventCount(),
@@ -426,7 +428,7 @@ async function generateCharts(context) {
 	if (config.makeChart && storage.eventData?.length > 0) {
 		const chartPath = typeof config.makeChart === 'string'
 			? config.makeChart
-			: `./${config.simulationName}-timeline`;
+			: `./${config.name}-timeline`;
 
 		await generateLineChart(storage.eventData, undefined, chartPath);
 
@@ -502,11 +504,13 @@ async function flushStorageToDisk(storage, config) {
 /**
  * Extract file information from storage containers
  * @param {import('./types').Storage} storage - Storage object
+ * @param {import('./types').Dungeon} config - Configuration object
  * @returns {string[]} Array of file paths
  */
-function extractFileInfo(storage) {
+async function extractFileInfo(storage, config) {
 	const files = [];
 
+	// Try to get paths from containers first
 	Object.values(storage).forEach(container => {
 		if (Array.isArray(container)) {
 			container.forEach(subContainer => {
@@ -519,6 +523,55 @@ function extractFileInfo(storage) {
 		}
 	});
 
+	// If no files found from containers and writeToDisk is enabled, scan the data directory
+	if (files.length === 0 && config.writeToDisk) {
+		try {
+			const fs = await import('fs');
+			const path = await import('path');
+			
+			let dataDir = path.resolve("./data");
+			if (!fs.existsSync(dataDir)) {
+				dataDir = path.resolve("./");
+			}
+			
+			if (fs.existsSync(dataDir)) {
+				const allFiles = fs.readdirSync(dataDir);
+				const simulationName = config.name;
+				
+				// Filter files that match our patterns and were likely created by this run
+				const relevantFiles = allFiles.filter(file => {
+					// Skip system files
+					if (file.startsWith('.')) return false;
+					
+					// If we have a simulation name, only include files with that prefix
+					if (simulationName && !file.startsWith(simulationName)) {
+						return false;
+					}
+					
+					// Check for common patterns
+					const hasEventPattern = file.includes('-EVENTS.');
+					const hasUserPattern = file.includes('-USERS.');
+					const hasScdPattern = file.includes('-SCD.');
+					const hasGroupPattern = file.includes('-GROUPS.');
+					const hasLookupPattern = file.includes('-LOOKUP.');
+					const hasAdspendPattern = file.includes('-ADSPEND.');
+					const hasMirrorPattern = file.includes('-MIRROR.');
+					
+					return hasEventPattern || hasUserPattern || hasScdPattern || 
+						   hasGroupPattern || hasLookupPattern || hasAdspendPattern || hasMirrorPattern;
+				});
+				
+				// Convert to full paths
+				relevantFiles.forEach(file => {
+					files.push(path.join(dataDir, file));
+				});
+			}
+		} catch (error) {
+			// If scanning fails, just return empty array
+			console.warn('Warning: Could not scan data directory for files:', error.message);
+		}
+	}
+
 	return files;
 }
 

package/lib/cli/cli.js

@@ -223,6 +223,12 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.option("name", {
+			alias: 'n',
+			demandOption: false,
+			describe: 'custom name for generated files (prefix)',
+			type: 'string'
+		})
 
 		.help()
 		.wrap(null)
@@ -236,7 +242,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 }
 
 
-function boolCoerce(value, foo) {
+function boolCoerce(value) {
 	if (typeof value === 'boolean') return value;
 	if (typeof value === 'string') {
 		return value.toLowerCase() === 'true';

package/lib/core/config-validator.js

@@ -136,9 +136,10 @@ export function validateDungeonConfig(config) {
 		throw new Error("Either epochStart or numDays must be provided");
 	}
 
-	// Generate simulation name
-	config.simulationName = name || makeName();
-	config.name = config.simulationName;
+	// Use provided name if non-empty string, otherwise generate one
+	if (!name || name === "") {
+		name = makeName();
+	}
 
 	// Validate events
 	if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
@@ -242,8 +243,7 @@ export function validateDungeonConfig(config) {
 		hasAndroidDevices,
 		hasDesktopDevices,
 		hasIOSDevices,
-		simulationName: config.simulationName,
-		name: config.name
+		name
 	};
 
 	return validatedConfig;

package/lib/core/context.js

@@ -40,6 +40,14 @@ function createDefaults(config, campaignData) {
 	const weighedBrowsers = u.weighArray(devices.browsers);
 	const weighedCampaigns = u.weighArray(campaignData);
 
+	// PERFORMANCE: Pre-compute device pools based on config to avoid rebuilding in makeEvent
+	const devicePools = {
+		android: config.hasAndroidDevices ? weighedAndroidDevices : [],
+		ios: config.hasIOSDevices ? weighedIOSDevices : [],
+		desktop: config.hasDesktopDevices ? weighedDesktopDevices : []
+	};
+	const allDevices = [...devicePools.android, ...devicePools.ios, ...devicePools.desktop];
+
 	return {
 		locationsUsers: () => weighedLocationsUsers,
 		locationsEvents: () => weighedLocationsEvents,
@@ -47,7 +55,11 @@ function createDefaults(config, campaignData) {
 		androidDevices: () => weighedAndroidDevices,
 		desktopDevices: () => weighedDesktopDevices,
 		browsers: () => weighedBrowsers,
-		campaigns: () => weighedCampaigns
+		campaigns: () => weighedCampaigns,
+		
+		// PERFORMANCE: Pre-computed device pools
+		devicePools,
+		allDevices
 	};
 }
 

package/lib/core/storage.js

@@ -65,13 +65,15 @@ export async function createHookArray(arr = [], opts) {
 	}
 
 	function getWritePath() {
+		const gzipSuffix = (config.gzip && !writeDir?.startsWith('gs://')) ? '.gz' : '';
+		
 		if (isBatchMode) {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
-			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}${gzipSuffix}`);
 		}
 		else {
-			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
-			return path.join(writeDir, `${filepath}.${format}`);
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}${gzipSuffix}`;
+			return path.join(writeDir, `${filepath}.${format}${gzipSuffix}`);
 		}
 	}
 
@@ -153,12 +155,19 @@ export async function createHookArray(arr = [], opts) {
 			console.log(`\n\twriting ${writePath}\n`);
 		}
 
+		const streamOptions = {
+			gzip: config.gzip || false
+		};
+
 		switch (format) {
 			case "csv":
-				writeResult = await u.streamCSV(writePath, data);
+				writeResult = await u.streamCSV(writePath, data, streamOptions);
 				break;
 			case "json":
-				writeResult = await u.streamJSON(writePath, data);
+				writeResult = await u.streamJSON(writePath, data, streamOptions);
+				break;
+			case "parquet":
+				writeResult = await u.streamParquet(writePath, data, streamOptions);
 				break;
 			default:
 				throw new Error(`format ${format} is not supported`);
@@ -219,12 +228,15 @@ export class StorageManager {
 	async initializeContainers() {
 		const { config } = this.context;
 
+		// Validate configuration for potential data loss scenarios
+		this.validateConfiguration(config);
+
 		/** @type {Storage} */
 		const storage = {
 			eventData: await createHookArray([], {
 				hook: config.hook,
 				type: "event",
-				filepath: `${config.simulationName || 'events'}-EVENTS`,
+				filepath: `${config.name}-EVENTS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -233,7 +245,7 @@ export class StorageManager {
 			userProfilesData: await createHookArray([], {
 				hook: config.hook,
 				type: "user",
-				filepath: `${config.simulationName || 'users'}-USERS`,
+				filepath: `${config.name}-USERS`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -242,7 +254,7 @@ export class StorageManager {
 			adSpendData: await createHookArray([], {
 				hook: config.hook,
 				type: "ad-spend",
-				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
+				filepath: `${config.name}-ADSPEND`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -255,7 +267,7 @@ export class StorageManager {
 			mirrorEventData: await createHookArray([], {
 				hook: config.hook,
 				type: "mirror",
-				filepath: `${config.simulationName || 'mirror'}-MIRROR`,
+				filepath: `${config.name}-MIRROR`,
 				format: config.format || "csv",
 				concurrency: config.concurrency || 1,
 				context: this.context
@@ -268,7 +280,7 @@ export class StorageManager {
 				const scdArray = await createHookArray([], {
 					hook: config.hook,
 					type: "scd",
-					filepath: `${config.simulationName || 'scd'}-${scdKey}-SCD`,
+					filepath: `${config.name}-${scdKey}-SCD`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -284,7 +296,7 @@ export class StorageManager {
 				const groupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "group",
-					filepath: `${config.simulationName || 'groups'}-${groupKey}-GROUPS`,
+					filepath: `${config.name}-${groupKey}-GROUPS`,
 					format: config.format || "csv",
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -300,7 +312,7 @@ export class StorageManager {
 				const lookupArray = await createHookArray([], {
 					hook: config.hook,
 					type: "lookup",
-					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
+					filepath: `${config.name}-${lookupConfig.key}-LOOKUP`,
 					format: "csv", // Always force CSV for lookup tables
 					concurrency: config.concurrency || 1,
 					context: this.context
@@ -312,4 +324,24 @@ export class StorageManager {
 
 		return storage;
 	}
+
+	/**
+	 * Validates configuration to prevent data loss scenarios
+	 * @param {Object} config - Configuration object
+	 */
+	validateConfiguration(config) {
+		// Check for potential data loss scenario: writeToDisk=false with low batchSize
+		if (config.writeToDisk === false) {
+			const batchSize = config.batchSize || 1_000_000;
+			const numEvents = config.numEvents || 0;
+			
+			if (batchSize < numEvents) {
+				throw new Error(
+					`Configuration error: writeToDisk is explicitly set to false but batchSize (${batchSize}) is lower than numEvents (${numEvents}). ` +
+					`This would result in data loss as batched data would be discarded. ` +
+					`Either set writeToDisk to true, increase batchSize to be >= numEvents, or provide a Mixpanel token to send data directly.`
+				);
+			}
+		}
+	}
 }

package/lib/generators/events.js

@@ -72,7 +72,6 @@ export async function makeEvent(
     };
 
     let defaultProps = {};
-    let devicePool = [];
 
     // Add default properties based on configuration
     if (hasLocation) {
@@ -82,32 +81,30 @@ export async function makeEvent(
     if (hasBrowser) {
         defaultProps.browser = u.choose(defaults.browsers());
     }
-    
-    // Build device pool based on enabled device types
-    if (hasAndroidDevices) devicePool.push(defaults.androidDevices());
-    if (hasIOSDevices) devicePool.push(defaults.iOSDevices());
-    if (hasDesktopDevices) devicePool.push(defaults.desktopDevices());
 
     // Add campaigns with attribution likelihood
     if (hasCampaigns && chance.bool({ likelihood: 25 })) {
         defaultProps.campaigns = u.pickRandom(defaults.campaigns());
     }
     
-    // Select device from pool
-    const devices = devicePool.flat();
-    if (devices.length) {
-        defaultProps.device = u.pickRandom(devices);
+    // PERFORMANCE: Use pre-computed device pool instead of rebuilding every time
+    if (defaults.allDevices.length) {
+        defaultProps.device = u.pickRandom(defaults.allDevices);
     }
 
     // Set event time using TimeSoup for realistic distribution
     if (earliestTime) {
         if (isFirstEvent) {
-            // Apply time shift to move to present day using precomputed value
-            eventTemplate.time = dayjs.unix(earliestTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Direct numeric calculation instead of dayjs object creation
+            const shiftedTimestamp = earliestTime + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         } else {
-            // Get time from TimeSoup and apply precomputed time shift
+            // Get time from TimeSoup (returns ISO string) and apply precomputed time shift
             const soupTime = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
-            eventTemplate.time = dayjs(soupTime).add(context.TIME_SHIFT_SECONDS, 'seconds').toISOString();
+            // PERFORMANCE: Parse ISO directly to milliseconds, add shift, convert back to ISO with one dayjs call
+            const soupTimestamp = new Date(soupTime).getTime() / 1000; // Convert to unix seconds
+            const shiftedTimestamp = soupTimestamp + context.TIME_SHIFT_SECONDS;
+            eventTemplate.time = dayjs.unix(shiftedTimestamp).toISOString();
         }
     }
 
@@ -133,7 +130,9 @@ export async function makeEvent(
     // PERFORMANCE: Process properties directly without creating intermediate object
     // Add custom properties from event configuration
     if (chosenEvent.properties) {
-        for (const key in chosenEvent.properties) {
+        const eventKeys = Object.keys(chosenEvent.properties);
+        for (let i = 0; i < eventKeys.length; i++) {
+            const key = eventKeys[i];
             try {
                 eventTemplate[key] = u.choose(chosenEvent.properties[key]);
             } catch (e) {
@@ -145,7 +144,9 @@ export async function makeEvent(
     
     // Add super properties (override event properties if needed)
     if (superProps) {
-        for (const key in superProps) {
+        const superKeys = Object.keys(superProps);
+        for (let i = 0; i < superKeys.length; i++) {
+            const key = superKeys[i];
             try {
                 eventTemplate[key] = u.choose(superProps[key]);
             } catch (e) {

package/lib/generators/funnels.js

@@ -128,7 +128,13 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
     
     return sequence.map((eventName) => {
         const foundEvent = config.events?.find((e) => e.event === eventName);
-        const eventSpec = u.deepClone(foundEvent) || { event: eventName, properties: {} };
+        
+        // PERFORMANCE: Shallow copy instead of deepClone for better performance
+        // We only need to copy the top-level structure since we're rebuilding properties anyway
+        const eventSpec = foundEvent ? {
+            event: foundEvent.event,
+            properties: { ...foundEvent.properties }
+        } : { event: eventName, properties: {} };
         
         // Process event properties
         for (const key in eventSpec.properties) {
@@ -139,11 +145,7 @@ function buildFunnelEvents(context, sequence, chosenFunnelProps) {
             }
         }
         
-        // Clean up funnel-specific properties
-        delete eventSpec.isFirstEvent;
-        delete eventSpec.weight;
-        
-        // Merge funnel properties
+        // Merge funnel properties (no need to delete properties since we're creating a new object)
         eventSpec.properties = { ...eventSpec.properties, ...chosenFunnelProps };
         
         return eventSpec;

package/lib/utils/utils.js

@@ -8,6 +8,7 @@ import utc from 'dayjs/plugin/utc.js';
 import path from 'path';
 import { mkdir, parseGCSUri } from 'ak-tools';
 import { existsSync } from 'fs';
+import zlib from 'zlib';
 dayjs.extend(utc);
 import 'dotenv/config';
 import { domainSuffix, domainPrefix } from '../templates/defaults.js';
@@ -483,15 +484,22 @@ STREAMERS
 ----
 */
 
-function streamJSON(filePath, data) {
+function streamJSON(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+		
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 		data.forEach(item => {
 			writeStream.write(JSON.stringify(item) + '\n');
@@ -504,15 +512,22 @@ function streamJSON(filePath, data) {
 	});
 }
 
-function streamCSV(filePath, data) {
+function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+		
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 
 		// Extract all unique keys from the data array
@@ -539,6 +554,117 @@ function streamCSV(filePath, data) {
 	});
 }
 
+async function streamParquet(filePath, data, options = {}) {
+	const { gzip = false } = options;
+	
+	// Dynamically import hyparquet-writer
+	const { parquetWriteFile, parquetWriteBuffer } = await import('hyparquet-writer');
+	
+	if (data.length === 0) {
+		throw new Error('Cannot write parquet file with empty data');
+	}
+
+	// Extract column names and data from the input array
+	const columns = getUniqueKeys(data);
+	const columnData = columns.map(columnName => {
+		const columnValues = data.map(row => {
+			let value = row[columnName];
+			
+			// Handle null/undefined values
+			if (value === null || value === undefined) {
+				return null;
+			}
+			
+			// Convert objects to strings
+			if (typeof value === 'object') {
+				value = JSON.stringify(value);
+			}
+			
+			return value;
+		});
+		
+		// Determine the type based on the first non-null value
+		let type = 'STRING'; // default
+		const firstValue = columnValues.find(v => v !== null && v !== undefined);
+		
+		if (firstValue !== undefined) {
+			if (typeof firstValue === 'boolean') {
+				type = 'BOOLEAN';
+			} else if (typeof firstValue === 'number') {
+				// For parquet compatibility, convert numbers to appropriate types
+				if (Number.isInteger(firstValue)) {
+					// Use INT32 for smaller integers, convert to BigInt for INT64 if needed
+					if (firstValue >= -2147483648 && firstValue <= 2147483647) {
+						type = 'INT32';
+					} else {
+						type = 'INT64';
+						// Convert all values to BigInt for INT64
+						for (let i = 0; i < columnValues.length; i++) {
+							if (columnValues[i] !== null && columnValues[i] !== undefined) {
+								columnValues[i] = BigInt(columnValues[i]);
+							}
+						}
+					}
+				} else {
+					type = 'DOUBLE';
+				}
+			} else if (firstValue instanceof Date) {
+				type = 'TIMESTAMP';
+			}
+		}
+		
+		return {
+			name: columnName,
+			data: columnValues,
+			type: type
+		};
+	});
+
+	if (filePath?.startsWith('gs://')) {
+		// For GCS, write to buffer first, then upload
+		const arrayBuffer = parquetWriteBuffer({ columnData });
+		const { bucket, file } = parseGCSUri(filePath);
+		
+		const writeStream = storage.bucket(bucket).file(file).createWriteStream({ 
+			gzip: gzip || true // Always gzip for GCS
+		});
+		
+		return new Promise((resolve, reject) => {
+			writeStream.write(Buffer.from(arrayBuffer));
+			writeStream.end();
+			writeStream.on('finish', () => resolve(filePath));
+			writeStream.on('error', reject);
+		});
+	} else {
+		// For local files
+		let actualFilePath = filePath;
+		if (gzip && !filePath.endsWith('.gz')) {
+			actualFilePath = filePath + '.gz';
+		}
+		
+		if (gzip) {
+			// Write to buffer then gzip to disk
+			const arrayBuffer = parquetWriteBuffer({ columnData });
+			const buffer = Buffer.from(arrayBuffer);
+			const gzippedBuffer = zlib.gzipSync(buffer);
+			
+			return new Promise((resolve, reject) => {
+				fs.writeFile(actualFilePath, gzippedBuffer, (err) => {
+					if (err) reject(err);
+					else resolve(actualFilePath);
+				});
+			});
+		} else {
+			// Direct write to disk
+			parquetWriteFile({
+				filename: filePath,
+				columnData
+			});
+			return Promise.resolve(filePath);
+		}
+	}
+}
+
 
 /*
 ----
@@ -870,7 +996,7 @@ function buildFileNames(config) {
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
-	let simName = config.simulationName;
+	let simName = config.name;
 	let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
 	if (config.writeToDisk) {
 		const dataFolder = path.resolve("./data");
@@ -1328,7 +1454,7 @@ export {
 	TimeSoup,
 	companyName,
 	generateEmoji,
-	hasSameKeys as haveSameKeys,
+	hasSameKeys,
 	deepClone,
 	initChance,
 	getChance,
@@ -1362,6 +1488,7 @@ export {
 	buildFileNames,
 	streamJSON,
 	streamCSV,
+	streamParquet,
 	datesBetween,
 	weighChoices,
 	wrapFunc,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "2.0.22",
+	"version": "2.0.23",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
@@ -70,6 +70,7 @@
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
 		"google-auth-library": "^9.15.0",
+		"hyparquet-writer": "^0.6.1",
 		"mixpanel-import": "^2.8.162",
 		"p-limit": "^3.1.0",
 		"yargs": "^17.7.2"

package/types.d.ts

@@ -20,7 +20,7 @@ export interface Dungeon {
     epochEnd?: number;
     numEvents?: number;
     numUsers?: number;
-    format?: "csv" | "json" | string;
+    format?: "csv" | "json" | "parquet" | string;
     region?: "US" | "EU";
     concurrency?: number;
     batchSize?: number;
@@ -30,7 +30,6 @@ export interface Dungeon {
     projectId?: string;
 
     // ids
-    simulationName?: string;
     name?: string;
 
     //switches
@@ -44,6 +43,7 @@ export interface Dungeon {
     hasDesktopDevices?: boolean;
     hasBrowser?: boolean;
     writeToDisk?: boolean | string;
+    gzip?: boolean;
     verbose?: boolean;
     hasAnonIds?: boolean;
     hasSessionIds?: boolean;
@@ -117,7 +117,7 @@ export interface hookArrayOptions<T> {
     hook?: Hook<T>;
     type?: hookTypes;
     filename?: string;
-    format?: "csv" | "json" | string;
+    format?: "csv" | "json" | "parquet" | string;
     concurrency?: number;
     context?: Context;
     [key: string]: any;