make-mp-data 1.3.2 → 1.3.4

package/.vscode/settings.json

@@ -23,5 +23,5 @@
 	],
 	"jest.runMode": "on-demand",
 	"jest.jestCommandLine": "npm run test --",
-	"js/ts.implicitProjectConfig.checkJs": false,
+	"js/ts.implicitProjectConfig.checkJs": true,
 }

package/cli.js

@@ -15,6 +15,7 @@ by ak@mixpanel.com
 
 function cliParams() {
 	console.log(hero);
+	// @ts-ignore
 	const args = yargs(process.argv.splice(2))
 		.scriptName("make-mp-data")
 		.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
@@ -78,17 +79,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'use complex data model (model all entities)',
 			alias: 'c',
 			type: 'boolean',
-			coerce: (value) => {
-				if (typeof value === 'boolean') return value;
-				if (value === 'true') {
-					return true;
-				}
-				if (value === 'false') {
-					return false;
-				}
-
-				return true;
-			}
+			coerce: boolCoerce
 		})
 		.option("writeToDisk", {
 			demandOption: false,
@@ -96,13 +87,25 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'write data to disk',
 			alias: 'w',
 			type: 'boolean',
-			coerce: (value) => {
-				if (typeof value === 'string') {
-					return value.toLowerCase() === 'true';
-				}
-				return value;
-			}
+			coerce: boolCoerce
+		})
+		.option("sessionIds", {
+			demandOption: false,
+			default: false,
+			describe: 'create session ids in the data',
+			alias: 'sid',
+			type: 'boolean',
+			coerce: boolCoerce
+		})
+		.option("anonIds", {
+			demandOption: false,
+			default: false,
+			describe: 'create anonymous ids in the data',
+			alias: 'aid',
+			type: 'boolean',
+			coerce: boolCoerce
 		})
+
 		.help()
 		.wrap(null)
 		.argv;
@@ -115,4 +118,14 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 }
 
 
+function boolCoerce(value, foo) {
+	if (typeof value === 'boolean') return value;
+	if (typeof value === 'string') {
+		return value.toLowerCase() === 'true';
+	}
+	return value;
+
+}
+
+
 module.exports = cliParams;

package/index.js

@@ -11,15 +11,13 @@ const mp = require("mixpanel-import");
 const path = require("path");
 const Chance = require("chance");
 const chance = new Chance();
-const { touch, comma, bytesHuman, mkdir } = require("ak-tools");
-const Papa = require("papaparse");
+const { comma, bytesHuman, mkdir, makeName, md5, clone, tracker, uid  } = require("ak-tools");
 const u = require("./utils.js");
 const AKsTimeSoup = require("./timesoup.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
-const { makeName, md5, clone, tracker, uid } = require('ak-tools');
 const NOW = dayjs().unix();
 let VERBOSE = false;
 let isCLI = false;
@@ -77,7 +75,6 @@ async function main(config) {
 		numEvents,
 		numUsers,
 		numDays,
-		events,
 		anonIds,
 		sessionIds,
 		format,
@@ -89,7 +86,7 @@ async function main(config) {
 	});
 	log(`------------------SETUP------------------`);
 	log(`\nyour data simulation will heretofore be known as: \n\n\t${simulationName.toUpperCase()}...\n`);
-	log(`and your configuration is:\n\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk }, null, 2));
+	log(`and your configuration is:\n\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk, anonIds, sessionIds }, null, 2));
 	log(`------------------SETUP------------------`, "\n");
 
 
@@ -117,11 +114,13 @@ async function main(config) {
 			const weight = event.weight || 1;
 			for (let i = 0; i < weight; i++) {
 
+				// @ts-ignore
 				acc.push(event);
 			}
 			return acc;
 		}, [])
 
+		// @ts-ignore
 		.filter((e) => !e.isFirstEvent);
 
 	const firstEvents = events.filter((e) => e.isFirstEvent);
@@ -257,7 +256,6 @@ async function main(config) {
 			numEvents,
 			numUsers,
 			numDays,
-			events,
 			anonIds,
 			sessionIds,
 			format,
@@ -278,7 +276,8 @@ async function main(config) {
 		};
 	}
 	log(`-----------------WRITES------------------`, `\n\n`);
-	//write the files
+
+	let writeFilePromises = [];
 	if (writeToDisk) {
 		if (verbose) log(`writing files... for ${simulationName}`);
 		loopFiles: for (const ENTITY of pairs) {
@@ -297,25 +296,16 @@ async function main(config) {
 				log(`\twriting ${path}`);
 				//if it's a lookup table, it's always a CSV
 				if (format === "csv" || path.includes("-LOOKUP.csv")) {
-					const columns = u.getUniqueKeys(TABLE);
-					//papa parse needs eac nested field JSON stringified
-					TABLE.forEach((e) => {
-						for (const key in e) {
-							if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
-						}
-					});
-
-					const csv = Papa.unparse(TABLE, { columns });
-					await touch(path, csv);
+					writeFilePromises.push(u.streamCSV(path, TABLE));
 				}
 				else {
-					const ndjson = TABLE.map((d) => JSON.stringify(d)).join("\n");
-					await touch(path, ndjson, false);
+					writeFilePromises.push(u.streamJSON(path, TABLE));
 				}
 
 			}
 		}
 	}
+	const fileWriteResults = await Promise.all(writeFilePromises);
 
 	const importResults = { events: {}, users: {}, groups: [] };
 
@@ -404,6 +394,10 @@ function makeProfile(props, defaults) {
 		...defaults,
 	};
 
+	// anonymous and session ids
+	if (!global.MP_SIMULATION_CONFIG?.anonIds) delete profile.anonymousIds;
+	if (!global.MP_SIMULATION_CONFIG?.sessionIds) delete profile.sessionIds;
+
 	for (const key in props) {
 		try {
 			profile[key] = u.choose(props[key]);
@@ -415,7 +409,7 @@ function makeProfile(props, defaults) {
 	return profile;
 }
 /**
- * @param  {import('./types.d.ts').valueValid} prop
+ * @param  {import('./types.d.ts').ValueValid} prop
  * @param  {string} scdKey
  * @param  {string} distinct_id
  * @param  {number} mutations
@@ -498,8 +492,11 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
 		const groupCardinality = groupPair[1];
+		const groupEvents = groupPair[2] || [];
 
-		event[groupKey] = u.pick(u.weightedRange(1, groupCardinality));
+		// empty array for group events means all events
+		if (!groupEvents.length) event[groupKey] = u.pick(u.weightedRange(1, groupCardinality));
+		if (groupEvents.includes(event.event)) event[groupKey] = u.pick(u.weightedRange(1, groupCardinality));
 	}
 
 	//make $insert_id
@@ -509,18 +506,21 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 }
 
 function buildFileNames(config) {
-	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
-	const extension = format === "csv" ? "csv" : "json";
+	const { format = "csv", groupKeys = [], lookupTables = [], m } = config;
+	let extension = "";
+	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
 	const simName = config.simulationName;
 	let writeDir = "./";
 	if (config.writeToDisk) writeDir = mkdir("./data");
+	if (typeof writeDir !== "string") throw new Error("writeDir must be a string");
+	if (typeof simName !== "string") throw new Error("simName must be a string");
 
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
 		scdFiles: [],
-		mirrorFiles: [path.join(writeDir, `${simName}-EVENTS-FUTURE-MIRROR.${extension}`)],
+		mirrorFiles: [],
 		groupFiles: [],
 		lookupFiles: [],
 		folder: writeDir,
@@ -534,13 +534,16 @@ function buildFileNames(config) {
 		);
 	}
 
+	//add group files
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
+
 		writePaths.groupFiles.push(
 			path.join(writeDir, `${simName}-${groupKey}-GROUP.${extension}`)
 		);
 	}
 
+	//add lookup files
 	for (const lookupTable of lookupTables) {
 		const { key } = lookupTable;
 		writePaths.lookupFiles.push(
@@ -549,10 +552,25 @@ function buildFileNames(config) {
 		);
 	}
 
+	//add mirror files
+	const mirrorProps = config?.mirrorProps || {};
+	if (Object.keys(mirrorProps).length) {
+		writePaths.mirrorFiles.push(
+			path.join(writeDir, `${simName}-MIRROR.${extension}`)
+		);
+	}
+
 	return writePaths;
 }
 
+/** @typedef {import('./types').EnrichedArray} EnrichArray */
+/** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
 
+/** 
+ * @param  {any[]} arr
+ * @param  {EnrichArrayOptions} opts
+ * @returns {EnrichArray}}
+ */
 function enrichArray(arr = [], opts = {}) {
 	const { hook = a => a, type = "", ...rest } = opts;
 
@@ -560,9 +578,15 @@ function enrichArray(arr = [], opts = {}) {
 		return arr.push(hook(item, type, rest));
 	}
 
-	arr.hPush = transformThenPush;
+	/** @type {EnrichArray} */
+	// @ts-ignore
+	const enrichedArray = arr;
+
+
+	enrichedArray.hPush = transformThenPush;
+	
 
-	return arr;
+	return enrichedArray;
 };
 
 
@@ -571,7 +595,9 @@ function enrichArray(arr = [], opts = {}) {
 if (require.main === module) {
 	isCLI = true;
 	const args = cliParams();
-	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false } = args;
+	// @ts-ignore
+	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false, sessionIds, anonIds } = args;
+	// @ts-ignore
 	const suppliedConfig = args._[0];
 
 	//if the user specifics an separate config file
@@ -605,6 +631,8 @@ if (require.main === module) {
 	if (region) config.region = region;
 	if (writeToDisk) config.writeToDisk = writeToDisk;
 	if (writeToDisk === 'false') config.writeToDisk = false;
+	if (sessionIds) config.sessionIds = sessionIds;
+	if (anonIds) config.anonIds = anonIds;
 	config.verbose = true;
 
 	main(config)

package/models/complex.js

@@ -63,6 +63,27 @@ const config = {
 
 			}
 		},
+		{
+			"event": "comment",
+			"weight": 2,
+			"properties": {
+				length: weightedRange(1, 500, 1000, .25),
+				video_id: weightedRange(1, 50000, 420000, 1.4),
+				has_replies: [true, false, false, false, false],
+				has_photo: [true, false, false, false, false],
+
+			}
+		},
+		{
+			"event": "save video",
+			"weight": 4,
+			"properties": {
+				video_id: weightedRange(1, 50000, 420000, 1.4),
+				ui_control: ["toolbar", "menu", "keyboard"]
+
+
+			}
+		},
 		{
 			"event": "view item",
 			"weight": 8,
@@ -143,8 +164,8 @@ const config = {
 	each pair represents a group_key and the number of profiles for that key
 	*/
 	groupKeys: [
-		['company_id', 500],
-		['room_id', 10000],
+		['company_id', 500, []],
+		['room_id', 10000, ["save video", "comment", "watch video"]],
 
 	],
 	groupProps: {
@@ -192,7 +213,6 @@ const config = {
 				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
 				uploader_id: chance.guid.bind(chance),
 				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
-				rating: weightedRange(1, 5),
 				thumbs: weightedRange(0, 35),
 				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
 			}
@@ -286,7 +306,7 @@ function deviceAttributes(isMobile = false) {
 		let devices = ["desktop", "laptop", "desktop", "laptop", "desktop", "laptop", "other"];
 		if (isMobile) devices = [...devices, "mobile", "mobile", "mobile", "tablet"];
 		const device = chance.pickone(devices);
-		const oses = ["Windows", "macOS", "Windows", "macOS", "macOS", "Linux", "Windows", "macOS", "Windows", "macOS", "macOS", "TempleOS"];
+		let oses = ["Windows", "macOS", "Windows", "macOS", "macOS", "Linux", "Windows", "macOS", "Windows", "macOS", "macOS", "TempleOS"];
 		if (isMobile) oses = [...oses, "iOS", "Android", "iOS", "Android"];
 		const os = chance.pickone(oses);
 		const browser = chance.pickone(["Chrome", "Firefox", "Safari", "Edge", "Opera", "IE", "Brave", "Vivaldi"]);

package/models/foobar.js

@@ -0,0 +1,110 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { weighList, weightedRange, date, integer } = require('../utils');
+
+const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
+
+const videoCategories = ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"];
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "foo bar",
+	numDays: 365, //how many days worth of data
+	numEvents: 10000000, //how many events
+	numUsers: 25000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	anonIds: true, //if true, anonymousIds are created for each user
+	sessionIds: false, //if true, sessionIds are created for each user
+
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		number: integer,
+
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weightedRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.3.02",
+  "version": "1.3.04",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",
@@ -46,5 +46,8 @@
     "mixpanel-import": "^2.5.51",
     "papaparse": "^5.4.1",
     "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12"
   }
 }

package/tests/unit.test.js

@@ -25,7 +25,7 @@ describe('timeSoup', () => {
 
 
 
-const { applySkew, boxMullerRandom, choose, date, dates, day, exhaust, generateEmoji, getUniqueKeys, integer, makeHashTags, makeProducts, mapToRange, person, pick, range, weighList, weightedRange } = require('../utils');
+const { applySkew, boxMullerRandom, choose, date, dates, day, exhaust, generateEmoji, getUniqueKeys, integer, mapToRange, person, pick, range, weighList, weightedRange } = require('../utils');
 
 describe('utils', () => {
 
@@ -124,8 +124,8 @@ describe('utils', () => {
 		const sd = 5;
 		const mappedValue = mapToRange(value, mean, sd);
 		expect(mappedValue).toBe(10);
-	  });
-	  
+	});
+
 
 
 	test('exhaust: elements', () => {
@@ -184,15 +184,24 @@ describe('utils', () => {
 	test('emoji: works', () => {
 		const emojis = generateEmoji(5)();
 		expect(typeof emojis).toBe('string');
-		expect(emojis.split(', ').length).toBeLessThanOrEqual(5);
+		if (!Array.isArray(emojis)) {
+			expect(emojis.split(', ').length).toBeLessThanOrEqual(5);
+		}
+		if (Array.isArray(emojis)) {
+			expect(emojis.length).toBeLessThanOrEqual(5);
+		}
 	});
 
 	test('emoji: length', () => {
 		const result = generateEmoji();
 		const emojis = result();
 		expect(typeof emojis).toBe('string');
-		const emojiArray = emojis.split(', ');
-		expect(emojiArray.length).toBeLessThanOrEqual(10); // Assuming max default is 10
+		if (!Array.isArray(emojis)) {
+			expect(emojis.split(', ').length).toBeLessThanOrEqual(10);
+		}
+		if (Array.isArray(emojis)) {
+			expect(emojis.length).toBeLessThanOrEqual(10);
+		}
 
 	});
 

package/timesoup.js

@@ -22,10 +22,10 @@ const PEAK_DAYS = [
 
 /**
  * essentially, a timestamp generator with a twist
- * @param {number} earliestTime - The earliest timestamp in Unix format.
- * @param {number} latestTime - The latest timestamp in Unix format.
- * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
- * @returns {number} - The generated event timestamp in Unix format.
+ * @param {number} [earliestTime] - The earliest timestamp in Unix format.
+ * @param {number} [latestTime] - The latest timestamp in Unix format.
+ * @param {Array} [peakDays] - Array of Unix timestamps representing the start of peak days.
+ * @returns {string} - The generated event timestamp in Unix format.
  */
 function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 	let chosenTime;

package/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "CommonJS",
+    "allowJs": true,
+    "checkJs": true,
+    "noEmit": true,
+    "strict": false,
+    "esModuleInterop": true,
+	"resolveJsonModule": true
+  },
+  "include": [
+    "**/*.js"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

package/types.d.ts

@@ -1,114 +1,128 @@
 declare namespace main {
-	type Primitives = string | number | boolean | Date | Record<string, any>;
-  
-	// Recursive type to handle functions returning functions that eventually return Primitives or arrays of Primitives
-	type ValueValid =
-	  | Primitives
-	  | ValueValid[]
-	  | (() => ValueValid);
-  
-	// MAIN CONFIGURATION OBJECT
-	export interface Config {
-	  token?: string;
-	  seed?: string;
-	  numDays?: number;
-	  numEvents?: number;
-	  numUsers?: number;
-	  format?: "csv" | "json";
-	  region?: string;
-	  events?: EventConfig[];
-	  superProps?: Record<string, ValueValid>;
-	  userProps?: Record<string, ValueValid>;
-	  scdProps?: Record<string, ValueValid>;
-	  mirrorProps?: Record<string, MirrorProps>;
-	  groupKeys?: [string, number][];
-	  groupProps?: Record<string, Record<string, ValueValid>>;
-	  lookupTables?: LookupTable[];
-	  writeToDisk?: boolean;
-	  simulationName?: string;
-	  verbose?: boolean;
-	  anonIds?: boolean;
-	  sessionIds?: boolean;
-	  hook?: Hook;
-	}
-  
-	export type Hook = (record: any, type: string, meta: any) => any;
-  
-	export interface EventConfig {
-	  event?: string;
-	  weight?: number;
-	  properties?: Record<string, ValueValid>;
-	  isFirstEvent?: boolean;
-	}
-  
-	export interface MirrorProps {
-	  events: string[] | "*";
-	  values: ValueValid[];
-	}
-  
-	export interface LookupTable {
-	  key: string;
-	  entries: number;
-	  attributes: Record<string, ValueValid>;
-	}
-  
-	export interface SCDTable {
-	  distinct_id: string;
-	  insertTime: string;
-	  startTime: string;
-	  [key: string]: ValueValid;
-	}
-  
-	export type Result = {
-	  eventData: EventData[];
-	  userProfilesData: any[];
-	  scdTableData: any[];
-	  groupProfilesData: GroupProfilesData[];
-	  lookupTableData: LookupTableData[];
-	  import?: ImportResults;
-	  files?: string[];
-	};
-  
-	export interface EventData {
-	  event: string;
-	  $source: string;
-	  time: string;
-	  $device_id?: string;
-	  $session_id?: string;
-	  $user_id?: string;
-	  [key: string]: any;
-	}
-  
-	export interface GroupProfilesData {
-	  key: string;
-	  data: any[];
-	}
-  
-	export interface LookupTableData {
-	  key: string;
-	  data: any[];
-	}
-  
-	export interface ImportResults {
-	  events: ImportResult;
-	  users: ImportResult;
-	  groups: ImportResult[];
-	}
-  
-	export interface ImportResult {
-	  success: number;
-	  bytes: number;
-	}
+  type Primitives = string | number | boolean | Date | Record<string, any>;
+
+  // Recursive type to handle functions returning functions that eventually return Primitives or arrays of Primitives
+  export type ValueValid = Primitives | ValueValid[] | (() => ValueValid);
+
+  // MAIN CONFIGURATION OBJECT
+  export interface Config {
+    token?: string;
+    seed?: string;
+    numDays?: number;
+    numEvents?: number;
+    numUsers?: number;
+    format?: "csv" | "json";
+    region?: "US" | "EU";
+    events?: EventConfig[];
+    superProps?: Record<string, ValueValid>;
+    userProps?: Record<string, ValueValid>;
+    scdProps?: Record<string, ValueValid>;
+    mirrorProps?: Record<string, MirrorProps>;
+    groupKeys?: [string, number][] | [string, number, string[]][];
+    groupProps?: Record<string, Record<string, ValueValid>>;
+    lookupTables?: LookupTable[];
+    writeToDisk?: boolean;
+    simulationName?: string;
+    verbose?: boolean;
+    anonIds?: boolean;
+    sessionIds?: boolean;
+    hook?: Hook<any>;
   }
-  
-  /**
-   * Mixpanel Data Generator
-   * model events, users, groups, and lookup tables (and SCD props!)
-   * @example
-   * const gen = require('make-mp-data')
-   * const dta = gen({writeToDisk: false})
-   */
-  declare function main(config: main.Config): Promise<main.Result>;
-  
-  export = main;
-  
+
+  export type Hook<T> = (record: any, type: string, meta: any) => T;
+
+  export interface EnrichArrayOptions<T> {
+    hook?: Hook<T>;
+    type?: string;
+    [key: string]: any;
+  }
+
+  export interface EnrichedArray<T> extends Array<T> {
+    hPush: (item: T) => number;
+  }
+
+  export interface EventConfig {
+    event?: string;
+    weight?: number;
+    properties?: Record<string, ValueValid>;
+    isFirstEvent?: boolean;
+  }
+
+  export interface MirrorProps {
+    events: string[] | "*";
+    values: ValueValid[];
+  }
+
+  export interface LookupTable {
+    key: string;
+    entries: number;
+    attributes: Record<string, ValueValid>;
+  }
+
+  export interface SCDTable {
+    distinct_id: string;
+    insertTime: string;
+    startTime: string;
+    [key: string]: ValueValid;
+  }
+
+  export type Result = {
+    eventData: EventData[];
+    userProfilesData: any[];
+    scdTableData: any[];
+    groupProfilesData: GroupProfilesData[];
+    lookupTableData: LookupTableData[];
+    import?: ImportResults;
+    files?: string[];
+  };
+
+  export interface EventData {
+    event: string;
+    $source: string;
+    time: string;
+    $device_id?: string;
+    $session_id?: string;
+    $user_id?: string;
+    [key: string]: any;
+  }
+
+  export interface GroupProfilesData {
+    key: string;
+    data: any[];
+  }
+
+  export interface LookupTableData {
+    key: string;
+    data: any[];
+  }
+
+  export interface ImportResults {
+    events: ImportResult;
+    users: ImportResult;
+    groups: ImportResult[];
+  }
+
+  export interface ImportResult {
+    success: number;
+    bytes: number;
+  }
+  export interface Person {
+    $name: string;
+    $email: string;
+    $avatar: string;
+    $created: string | undefined;
+    anonymousIds: string[];
+    sessionIds: string[];
+  }
+}
+
+/**
+ * Mixpanel Data Generator
+ * model events, users, groups, and lookup tables (and SCD props!)
+ * @example
+ * const gen = require('make-mp-data')
+ * const dta = gen({writeToDisk: false})
+ */
+declare function main(config: main.Config): Promise<main.Result>;
+
+export = main;

package/utils.js

@@ -1,3 +1,5 @@
+const fs = require('fs');
+const Papa = require('papaparse');
 const Chance = require('chance');
 const chance = new Chance();
 const readline = require('readline');
@@ -5,8 +7,10 @@ const { comma, uid } = require('ak-tools');
 const { spawn } = require('child_process');
 const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
+
 dayjs.extend(utc);
 
+
 function pick(items) {
 	if (!Array.isArray(items)) {
 		if (typeof items === 'function') {
@@ -57,10 +61,9 @@ function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
 function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
 	const pairs = [];
 	for (let i = 0; i < numPairs; i++) {
-		pairs.push([date(inTheLast, format), date(inTheLast, format)]);
+		pairs.push([date(inTheLast, true, format), date(inTheLast, true, format)]);
 	}
 	return pairs;
-
 };
 
 function day(start, end) {
@@ -96,11 +99,15 @@ function choose(value) {
 			return value;
 		}
 
+		if (typeof value === 'number') {
+			return value;
+		}
+
 		// If it's not a function or array, return it as is
 		return value;
 	}
 	catch (e) {
-		console.error(`\n\nerror on value: ${value};\n\n`,e, '\n\n');
+		console.error(`\n\nerror on value: ${value};\n\n`, e, '\n\n');
 		return '';
 	}
 }
@@ -111,7 +118,7 @@ function exhaust(arr) {
 };
 
 
-function integer(min, max) {
+function integer(min = 1, max = 100) {
 	if (min === max) {
 		return min;
 	}
@@ -178,22 +185,23 @@ function unOptimizedWeightedRange(min, max, size = 100, skew = 1) {
 
 // optimized weighted range
 function weightedRange(min, max, size = 100, skew = 1) {
-    const mean = (max + min) / 2;
-    const sd = (max - min) / 4;
-    const array = [];
-    while (array.length < size) {
-        const normalValue = boxMullerRandom();
-        const skewedValue = applySkew(normalValue, skew);
-        const mappedValue = mapToRange(skewedValue, mean, sd);
-        if (mappedValue >= min && mappedValue <= max) {
-            array.push(mappedValue);
-        }
-    }
-    return array;
+	const mean = (max + min) / 2;
+	const sd = (max - min) / 4;
+	const array = [];
+	while (array.length < size) {
+		const normalValue = boxMullerRandom();
+		const skewedValue = applySkew(normalValue, skew);
+		const mappedValue = mapToRange(skewedValue, mean, sd);
+		if (mappedValue >= min && mappedValue <= max) {
+			array.push(mappedValue);
+		}
+	}
+	return array;
 }
 
 
 function progress(thing, p) {
+	// @ts-ignore
 	readline.cursorTo(process.stdout, 0);
 	process.stdout.write(`${thing} processed ... ${comma(p)}`);
 };
@@ -226,7 +234,12 @@ function getUniqueKeys(data) {
 	return Array.from(keysSet);
 };
 
-//makes a random-sized array of emojis
+//
+/**
+ * makes a random-sized array of emojis
+ * @param  {number} max=10
+ * @param  {boolean} array=false
+ */
 function generateEmoji(max = 10, array = false) {
 	return function () {
 		const emojis = ['😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😈', '👿', '👦', '👧', '👨', '👩', '👴', '👵', '👶', '🧒', '👮', '👷', '💂', '🕵', '👩‍⚕️', '👨‍⚕️', '👩‍🌾', '👨‍🌾', '👩‍🍳', '👨‍🍳', '👩‍🎓', '👨‍🎓', '👩‍🎤', '👨‍🎤', '👩‍🏫', '👨‍🏫', '👩‍🏭', '👨‍🏭', '👩‍💻', '👨‍💻', '👩‍💼', '👨‍💼', '👩‍🔧', '👨‍🔧', '👩‍🔬', '👨‍🔬', '👩‍🎨', '👨‍🎨', '👩‍🚒', '👨‍🚒', '👩‍✈️', '👨‍✈️', '👩‍🚀', '👨‍🚀', '👩‍⚖️', '👨‍⚖️', '🤶', '🎅', '👸', '🤴', '👰', '🤵', '👼', '🤰', '🙇', '💁', '🙅', '🙆', '🙋', '🤦', '🤷', '🙎', '🙍', '💇', '💆', '🕴', '💃', '🕺', '🚶', '🏃', '🤲', '👐', '🙌', '👏', '🤝', '👍', '👎', '👊', '✊', '🤛', '🤜', '🤞', '✌️', '🤟', '🤘', '👌', '👈', '👉', '👆', '👇', '☝️', '✋', '🤚', '🖐', '🖖', '👋', '🤙', '💪', '🖕', '✍️', '🤳', '💅', '👂', '👃', '👣', '👀', '👁', '🧠', '👅', '👄', '💋', '👓', '🕶', '👔', '👕', '👖', '🧣', '🧤', '🧥', '🧦', '👗', '👘', '👙', '👚', '👛', '👜', '👝', '🛍', '🎒', '👞', '👟', '👠', '👡', '👢', '👑', '👒', '🎩', '🎓', '🧢', '⛑', '📿', '💄', '💍', '💎', '🔇', '🔈', '🔉', '🔊', '📢', '📣', '📯', '🔔', '🔕', '🎼', '🎵', '🎶', '🎙', '🎚', '🎛', '🎤', '🎧', '📻', '🎷', '🎸', '🎹', '🎺', '🎻', '🥁', '📱', '📲', '💻', '🖥', '🖨', '🖱', '🖲', '🕹', '🗜', '💽', '💾', '💿', '📀', '📼', '📷', '📸', '📹', '🎥', '📽', '🎞', '📞', '☎️', '📟', '📠', '📺', '📻', '🎙', '📡', '🔍', '🔎', '🔬', '🔭', '📡', '💡', '🔦', '🏮', '📔', '📕', '📖', '📗', '📘', '📙', '📚', '📓', '📒', '📃', '📜', '📄', '📰', '🗞', '📑', '🔖', '🏷', '💰', '💴', '💵', '💶', '💷', '💸', '💳', '🧾', '💹', '💱', '💲', '✉️', '📧', '📨', '📩', '📤', '📥', '📦', '📫', '📪', '📬', '📭', '📮', '🗳', '✏️', '✒️', '🖋', '🖊', '🖌', '🖍', '📝', '💼', '📁', '📂', '🗂', '📅', '📆', '🗒', '🗓', '📇', '📈', '📉', '📊', '📋', '📌', '📍', '📎', '🖇', '📏', '📐', '✂️', '🗃', '🗄', '🗑', '🔒', '🔓', '🔏', '🔐', '🔑', '🗝', '🔨', '⛏', '⚒', '🛠', '🗡', '⚔️', '🔫', '🏹', '🛡', '🔧', '🔩', '⚙️', '🗜', '⚖️', '🔗', '⛓', '🧰', '🧲', '⚗️', '🧪', '🧫', '🧬', '🔬', '🔭', '📡', '💉', '💊', '🛏', '🛋', '🚪', '🚽', '🚿', '🛁', '🧴', '🧷', '🧹', '🧺', '🧻', '🧼', '🧽', '🧯', '🚬', '⚰️', '⚱️', '🗿', '🏺', '🧱', '🎈', '🎏', '🎀', '🎁', '🎊', '🎉', '🎎', '🏮', '🎐', '🧧', '✉️', '📩', '📨', '📧'];
@@ -241,10 +254,17 @@ function generateEmoji(max = 10, array = false) {
 	};
 };
 
+/** @typedef {import('./types').Person} Person */
 
+/**
+ * @param  {number} bornDaysAgo=30
+ * @return {Person}
+ */
 function person(bornDaysAgo = 30) {
 	//names and photos
-	const gender = chance.pickone(['male', 'female']);
+	let gender = chance.pickone(['male', 'female']);
+	if (!gender) gender = "female";
+	// @ts-ignore
 	const first = chance.first({ gender });
 	const last = chance.last();
 	const $name = `${first} ${last}`;
@@ -256,33 +276,35 @@ function person(bornDaysAgo = 30) {
 	});
 	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
 	const $avatar = avatarPrefix + avPath;
-	const $created = date(bornDaysAgo, true, null)();
+	const $created = date(bornDaysAgo, true)();
 
+	/** @type {Person} */
 	const user = {
 		$name,
 		$email,
 		$avatar,
 		$created,
+		anonymousIds: [],
+		sessionIds: []
 	};
 
 	//anon Ids
 	if (global.MP_SIMULATION_CONFIG?.anonIds) {
-		const anonymousIds = [];
 		const clusterSize = integer(2, 10);
 		for (let i = 0; i < clusterSize; i++) {
-			anonymousIds.push(uid(42));
+			const anonId = uid(42);
+			user.anonymousIds.push(anonId);
 		}
-		user.anonymousIds = anonymousIds;
+
 	}
 
 	//session Ids
 	if (global.MP_SIMULATION_CONFIG?.sessionIds) {
-		const sessionIds = [];
 		const sessionSize = integer(5, 30);
 		for (let i = 0; i < sessionSize; i++) {
-			sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
+			const sessionId = [uid(5), uid(5), uid(5), uid(5)].join("-");
+			user.sessionIds.push(sessionId);
 		}
-		user.sessionIds = sessionIds;
 	}
 
 	return user;
@@ -316,6 +338,51 @@ function weighList(items, mostChosenIndex) {
 	};
 }
 
+
+
+
+function streamJSON(path, data) {
+	return new Promise((resolve, reject) => {
+		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		data.forEach(item => {
+			writeStream.write(JSON.stringify(item) + '\n');
+		});
+		writeStream.end();
+		writeStream.on('finish', () => {
+			resolve(path);
+		});
+		writeStream.on('error', reject);
+	});
+}
+
+function streamCSV(path, data) {
+	return new Promise((resolve, reject) => {
+		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		// Extract all unique keys from the data array
+		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
+
+		// Stream the header
+		writeStream.write(columns.join(',') + '\n');
+
+		// Stream each data row
+		data.forEach(item => {
+			for (const key in item) {
+				// Ensure all nested objects are properly stringified
+				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
+			}
+			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
+			writeStream.write(row + '\n');
+		});
+
+		writeStream.end();
+		writeStream.on('finish', () => {
+			resolve(path);
+		});
+		writeStream.on('error', reject);
+	});
+}
+
+
 module.exports = {
 	pick,
 	date,
@@ -335,5 +402,9 @@ module.exports = {
 	getUniqueKeys,
 	generateEmoji,
 	person,
-	weighList
+	weighList,
+
+
+	streamJSON,
+	streamCSV
 };