make-mp-data 1.2.25 → 1.3.0

package/.vscode/launch.json

@@ -4,6 +4,18 @@
 	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
 	"version": "0.2.0",
 	"configurations": [
+		{
+			"command": "npm run simple",
+			"name": "simple",
+			"request": "launch",
+			"type": "node-terminal"
+		},
+		{
+			"command": "npm run complex",
+			"name": "complex",
+			"request": "launch",
+			"type": "node-terminal"
+		},			
 		{
 			"type": "node",
 			"request": "launch",
@@ -15,5 +27,6 @@
 			"internalConsoleOptions": "neverOpen",
 			"args": ["--ignore", "./data/"]
 		}
+
 	]
 }

package/README.md

@@ -1,50 +1,103 @@
 
-
 # Make Mixpanel Data
-a quick and dirty CLI in node.js to generate fake data for mixpanel.
 
-## tldr;
+## 🤖 What is this?
+
+Generate fake Mixpanel data _quickly_ and _easily_ with a simple CLI in Node.js. **Create events, user profiles, groups profiles, SCD data, mirror tables, and lookup tables**; basically everything you need for testing + demoing mixpanel analysis features. 
+
+the CLI looks like this:
+
+![Generate Mixpanel data](https://aktunes.neocities.org/makeDataDemo.gif)
+
+under the hood, `make-mp-data` is modeling data adherent to match [Mixpanel's data model](https://docs.mixpanel.com/docs/data-structure/concepts), giving you the tools you need for robust, realistical field ready test data.
+
+## 🚀 Quick Start
+
+### Basic Usage
+
+Generate events and users, and write them to CSV files:
 
- 
- ```bash
+```bash
 npx make-mp-data 
 ```
- - makes events + users (and writes them to CSVs)
 
- ```bash
-npx make-mp-data --numUsers 100 --numEvents 10000 --days 90 --format json
+### Customizing Output
+
+Generate 10,000 events and 100 users over the last 90 days, and write the output as JSON:
+
+```bash
+npx make-mp-data --numUsers 100 --numEvents 10000 --numDays 90 --format json
 ```
- - makes ~10k events + 100 users from the last 90 days (but writes JSON)
 
+### Complex Data Models
 
- ```bash
+Create a comprehensive dataset including events, users, groups, SCD, and lookup tables:
+
+```bash
 npx make-mp-data --complex
 ```
- - makes events + users + groups + scd + lookup tables
- - this includes every type of data that mixpanel supports
+
+### Send Data to Mixpanel
+
+Generate and send data directly to Mixpanel using your project token:
 
 ```bash
 npx make-mp-data --token 1234
 ```
- - makes events + users (and send them to mixpanel)
+
+### Help and Options
+
+Need more info on available options?
 
 ```bash
 npx make-mp-data --help
 ```
-- explains all the options you can specify
 
-## customization
+## 🔧 Schema
+
+To choose your own event and property names and values, specify a custom data model and additional options:
 
 ```bash
 npx make-mp-data [dataModel.js] [options]
 ```
-ex.
+
+Example:
 
 ```bash
 npx make-mp-data ecommSpec.js --token 1234 --numDays 30 --numUsers 1000 --numEvents 1000000
 ```
 
-see `--help` for a full list of options
+### Data Models
+
+Check out `./models/` for example `dataModel.js` files to get started quickly.
+
+## 🛠️ CLI Options
+
+Here's a breakdown of the CLI options you can use with `make-mp-data`:
+
+- `--numUsers`: Number of users to generate.
+- `--numEvents`: Number of events to generate.
+- `--numDays`: Number of days over which to spread the generated data.
+- `--format`: Output format (`csv` or `json`).
+- `--token`: Mixpanel project token for direct data import.
+- `--region`: Mixpanel data region (`US`, `EU`).
+- `--writeToDisk`: Whether to write the data to disk (`true` or `false`).
+- `--verbose`: Enable verbose logging.
+- `--complex`: create a complex set models including groups, SCD, and lookup tables.
+- `--simple`: create a simple dataset including events, and users
+
+## 📄 Examples
+
+Check out the examples directory for sample data models:
+
+```bash
+ls ./models/
+```
+
+These models provide a great starting point for creating your own custom data generation scripts.
+
+## 🤝 Contributing
 
-see `./models/` for a few `dataModel.js` examples...
+Feel free to fork this repository and submit pull requests. Contributions are always welcome!
 
+For any issues or feature requests, please create an issue on the [GitHub repository](https://github.com/ak--47/make-mp-data/issues).

package/index.js

@@ -1,6 +1,5 @@
 #! /usr/bin/env node
 
-
 /*
 make fake mixpanel data easily!
 by AK 
@@ -20,9 +19,16 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
-const { makeName, md5 } = require('ak-tools');
+const { makeName, md5, clone, tracker, uid } = require('ak-tools');
 const NOW = dayjs().unix();
 let VERBOSE = false;
+let isCLI = false;
+
+const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41");
+function track() {
+	if (process.env.NODE_ENV === 'test') return;
+	metrics.track(...arguments);
+}
 
 /** @typedef {import('./types.d.ts').Config} Config */
 /** @typedef {import('./types.d.ts').EventConfig} EventConfig */
@@ -43,12 +49,13 @@ async function main(config) {
 			favoriteColor: ["red", "green", "blue", "yellow"],
 			spiritAnimal: chance.animal.bind(chance),
 		},
-		scdProps = { NPS: u.weightedRange(0, 10, 150, 1.6) },
+		scdProps = {},
+		mirrorProps = {},
 		groupKeys = [],
 		groupProps = {},
 		lookupTables = [],
-		anonIds = true,
-		sessionIds = true,
+		anonIds = false,
+		sessionIds = false,
 		format = "csv",
 		token = null,
 		region = "US",
@@ -58,10 +65,27 @@ async function main(config) {
 	} = config;
 	VERBOSE = verbose;
 	config.simulationName = makeName();
+	const { simulationName } = config;
 	global.MP_SIMULATION_CONFIG = config;
 	const uuidChance = new Chance(seed);
+	const runId = uid(42);
+	track('start simulation', {
+		runId,
+		seed,
+		numEvents,
+		numUsers,
+		numDays,
+		events,
+		anonIds,
+		sessionIds,
+		format,
+		token,
+		region,
+		writeToDisk,
+		isCLI
+	});
 	log(`------------------SETUP------------------`);
-	log(`\nyour data simulation will heretofore be known as: \n\n\t${config.simulationName.toUpperCase()}...\n`);
+	log(`\nyour data simulation will heretofore be known as: \n\n\t${simulationName.toUpperCase()}...\n`);
 	log(`and your configuration is:\n\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk }, null, 2));
 	log(`------------------SETUP------------------`, "\n");
 
@@ -100,7 +124,12 @@ async function main(config) {
 	const firstEvents = events.filter((e) => e.isFirstEvent);
 	const eventData = enrichArray([], { hook, type: "event", config });
 	const userProfilesData = enrichArray([], { hook, type: "user", config });
-	let scdTableData = enrichArray([], { hook, type: "scd", config });
+	const scdTableKeys = Object.keys(scdProps);
+	const scdTableData = [];
+	for (const [index, key] of scdTableKeys.entries()) {
+		scdTableData[index] = enrichArray([], { hook, type: "scd", config, scdKey: key });
+	}
+	// const scdTableData = enrichArray([], { hook, type: "scd", config });
 	const groupProfilesData = enrichArray([], { hook, type: "groups", config });
 	const lookupTableData = enrichArray([], { hook, type: "lookups", config });
 	const avgEvPerUser = Math.floor(numEvents / numUsers);
@@ -112,8 +141,13 @@ async function main(config) {
 		const user = generateUser();
 		const { distinct_id, $created, anonymousIds, sessionIds } = user;
 		userProfilesData.hPush(makeProfile(userProps, user));
-		const mutations = chance.integer({ min: 1, max: 10 });
-		scdTableData.hPush(makeSCD(scdProps, distinct_id, mutations, $created));
+
+		//scd loop
+		for (const [index, key] of scdTableKeys.entries()) {
+			const mutations = chance.integer({ min: 1, max: 10 });
+			scdTableData[index].hPush(makeSCD(scdProps[key], key, distinct_id, mutations, $created));
+		}
+
 		const numEventsThisUser = Math.round(
 			chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / u.integer(3, 7) })
 		);
@@ -150,7 +184,7 @@ async function main(config) {
 	}
 
 	//flatten SCD
-	scdTableData = scdTableData.flat();
+	scdTableData.forEach((table, index) => scdTableData[index] = table.flat());
 
 	log("\n");
 
@@ -186,25 +220,56 @@ async function main(config) {
 		}
 		lookupTableData.hPush({ key, data });
 	}
-	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder } =
+
+	// deal with mirror props
+	let mirrorEventData = [];
+	const mirrorPropKeys = Object.keys(mirrorProps);
+	if (mirrorPropKeys.length) {
+		mirrorEventData = clone(eventData);
+		for (const row of mirrorEventData) {
+			for (const key of mirrorPropKeys) {
+				if (mirrorProps[key]?.events?.includes(row?.event)) row[key] = hook(u.choose(mirrorProps[key]?.values), "mirror", { config, row, key });
+				if (mirrorProps[key]?.events === "*") row[key] = hook(u.choose(mirrorProps[key]?.values), "mirror", { config, row, key });
+			}
+		}
+	}
+
+	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder } =
 		buildFileNames(config);
 	const pairs = [
-		[eventFiles, eventData],
-		[userFiles, userProfilesData],
+		[eventFiles, [eventData]],
+		[userFiles, [userProfilesData]],
 		[scdFiles, scdTableData],
 		[groupFiles, groupProfilesData],
 		[lookupFiles, lookupTableData],
+		[mirrorFiles, [mirrorEventData]],
 	];
 	log("\n");
 	log(`---------------SIMULATION----------------`, "\n");
 
 	if (!writeToDisk && !token) {
+		track('end simulation', {
+			runId,
+			seed,
+			numEvents,
+			numUsers,
+			numDays,
+			events,
+			anonIds,
+			sessionIds,
+			format,
+			token,
+			region,
+			writeToDisk,
+			isCLI
+		});
 		return {
 			eventData,
 			userProfilesData,
 			scdTableData,
 			groupProfilesData,
 			lookupTableData,
+			mirrorEventData,
 			import: {},
 			files: []
 		};
@@ -212,32 +277,39 @@ async function main(config) {
 	log(`-----------------WRITES------------------`, `\n\n`);
 	//write the files
 	if (writeToDisk) {
-		if (verbose) log(`writing files... for ${config.simulationName}`);
-		loopFiles: for (const pair of pairs) {
-			const [paths, data] = pair;
+		if (verbose) log(`writing files... for ${simulationName}`);
+		loopFiles: for (const ENTITY of pairs) {
+			const [paths, data] = ENTITY;
 			if (!data.length) continue loopFiles;
-			for (const path of paths) {
-				let datasetsToWrite;
-				if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
-				else datasetsToWrite = [data];
-				for (const writeData of datasetsToWrite) {
-					//if it's a lookup table, it's always a CSV
-					if (format === "csv" || path.includes("-LOOKUP.csv")) {
-						log(`\twriting ${path}`);
-						const columns = u.getUniqueKeys(writeData);
-						//papa parse needs nested JSON stringified
-						writeData.forEach((e) => {
-							for (const key in e) {
-								if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
-							}
-						});
-						const csv = Papa.unparse(writeData, { columns });
-						await touch(path, csv);
-					} else {
-						const ndjson = data.map((d) => JSON.stringify(d)).join("\n");
-						await touch(path, ndjson, false);
-					}
+			for (const [index, path] of paths.entries()) {
+				let TABLE;
+				//group + lookup tables are structured differently
+				if (data?.[index]?.["key"]) {
+					TABLE = data[index].data;
+				}
+				else {
+					TABLE = data[index];
 				}
+
+				log(`\twriting ${path}`);
+				//if it's a lookup table, it's always a CSV
+				if (format === "csv" || path.includes("-LOOKUP.csv")) {
+					const columns = u.getUniqueKeys(TABLE);
+					//papa parse needs eac nested field JSON stringified
+					TABLE.forEach((e) => {
+						for (const key in e) {
+							if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
+						}
+					});
+
+					const csv = Papa.unparse(TABLE, { columns });
+					await touch(path, csv);
+				}
+				else {
+					const ndjson = TABLE.map((d) => JSON.stringify(d)).join("\n");
+					await touch(path, ndjson, false);
+				}
+
 			}
 		}
 	}
@@ -298,9 +370,24 @@ async function main(config) {
 
 	}
 	log(`\n-----------------WRITES------------------`, "\n");
+	track('end simulation', {
+		runId,
+		seed,
+		numEvents,
+		numUsers,
+		numDays,
+		events,
+		anonIds,
+		sessionIds,
+		format,
+		token,
+		region,
+		writeToDisk,
+		isCLI
+	});
 	return {
 		import: importResults,
-		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder],
+		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder],
 	};
 }
 
@@ -323,16 +410,23 @@ function makeProfile(props, defaults) {
 
 	return profile;
 }
-
-function makeSCD(props, distinct_id, mutations, $created) {
-	if (JSON.stringify(props) === "{}") return [];
+/**
+ * @param  {import('./types.d.ts').valueValid} prop
+ * @param  {string} scdKey
+ * @param  {string} distinct_id
+ * @param  {number} mutations
+ * @param  {string} $created
+ */
+function makeSCD(prop, scdKey, distinct_id, mutations, $created) {
+	if (JSON.stringify(prop) === "{}") return {};
+	if (JSON.stringify(prop) === "[]") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs($created);
 	const deltaDays = dayjs().diff(lastInserted, "day");
 
 	for (let i = 0; i < mutations; i++) {
 		if (lastInserted.isAfter(dayjs())) break;
-		const scd = makeProfile(props, { distinct_id });
+		const scd = makeProfile({ [scdKey]: prop }, { distinct_id });
 		scd.startTime = lastInserted.toISOString();
 		lastInserted = lastInserted.add(u.integer(1, 1000), "seconds");
 		scd.insertTime = lastInserted.toISOString();
@@ -421,12 +515,21 @@ function buildFileNames(config) {
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
-		scdFiles: [path.join(writeDir, `${simName}-SCD.${extension}`)],
+		scdFiles: [],
+		mirrorFiles: [path.join(writeDir, `${simName}-EVENTS-FUTURE-MIRROR.${extension}`)],
 		groupFiles: [],
 		lookupFiles: [],
 		folder: writeDir,
 	};
 
+	//add SCD files
+	const scdKeys = Object.keys(config?.scdProps || {});
+	for (const key of scdKeys) {
+		writePaths.scdFiles.push(
+			path.join(writeDir, `${simName}-${key}-SCD.${extension}`)
+		);
+	}
+
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
 		writePaths.groupFiles.push(
@@ -447,10 +550,10 @@ function buildFileNames(config) {
 
 
 function enrichArray(arr = [], opts = {}) {
-	const { hook = a => a, type = "", config = {} } = opts;
+	const { hook = a => a, type = "", ...rest } = opts;
 
 	function transformThenPush(item) {
-		return arr.push(hook(item, type, config));
+		return arr.push(hook(item, type, rest));
 	}
 
 	arr.hPush = transformThenPush;
@@ -462,6 +565,7 @@ function enrichArray(arr = [], opts = {}) {
 
 // this is for CLI
 if (require.main === module) {
+	isCLI = true;
 	const args = cliParams();
 	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false } = args;
 	const suppliedConfig = args._[0];
@@ -543,5 +647,13 @@ if (require.main === module) {
 
 
 function log(...args) {
+	const cwd = process.cwd();  // Get the current working directory
+
+	for (let i = 0; i < args.length; i++) {
+		// Replace occurrences of the current working directory with "./" in string arguments
+		if (typeof args[i] === 'string') {
+			args[i] = args[i].replace(new RegExp(cwd, 'g'), ".");
+		}
+	}
 	if (VERBOSE) console.log(...args);
 }

package/models/complex.js

@@ -1,5 +1,5 @@
 /**
- * This is the default configuration file for the data generator
+ * This is the default configuration file for the data generator in COMPLEX mode
  * notice how the config object is structured, and see it's type definition in ./types.d.ts
  * feel free to modify this file to customize the data you generate
  * see helper functions in utils.js for more ways to generate data
@@ -59,7 +59,7 @@ const config = {
 				watchTimeSec: weightedRange(10, 600, 1000, .25),
 				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
 				format: ["mp4", "avi", "mov", "mpg"],
-				uploader_id: chance.guid.bind(chance)
+				video_id: weightedRange(1, 50000, 420000, 1.4),
 
 			}
 		},
@@ -120,12 +120,22 @@ const config = {
 
 	},
 
+	/** each generates it's own table */
 	scdProps: {
 		plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
 		MRR: weightedRange(0, 10000, 1000, .15),
 		NPS: weightedRange(0, 10, 150, 2),
-		marketingOptIn: [true, true, false],
-		dateOfRenewal: date(100, false),
+		subscribed: [true, true, true, true, true, true, false, false, false, false, "it's complicated"],
+		renewalDate: date(100, false),
+	},
+
+	mirrorProps: {
+		isBot: { events: "*", values: [false, false, false, false, true] },
+		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
+		watchTimeSec: {
+			events: ["watch video"],
+			values: weightedRange(50, 1200, 247, 6)
+		}
 	},
 
 	/*
@@ -133,7 +143,8 @@ const config = {
 	each pair represents a group_key and the number of profiles for that key
 	*/
 	groupKeys: [
-		['company_id', 350],
+		['company_id', 500],
+		['room_id', 10000],
 
 	],
 	groupProps: {
@@ -144,6 +155,15 @@ const config = {
 			"industry": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
 			"segment": ["enterprise", "SMB", "mid-market"],
 			"products": [["core"], ["core"], ["core", "add-ons"], ["core", "pro-serve"], ["core", "add-ons", "pro-serve"], ["core", "BAA", "enterprise"], ["free"], ["free"], ["free", "addons"]],
+		},
+		room_id: {
+			$name: () => { return `#${chance.word({ length: integer(4, 24), capitalize: true })}`; },
+			$email: ["public", "private"],
+			"room provider": ["partner", "core", "core", "core"],
+			"room capacity": weightedRange(3, 1000000),
+			"isPublic": [true, false, false, false, false],
+			"country": chance.country.bind(chance),
+			"isVerified": [true, true, false, false, false],
 		}
 	},
 
@@ -153,29 +173,7 @@ const config = {
 			entries: 1000,
 			attributes: {
 				category: [
-					"Books",
-					"Movies",
-					"Music",
-					"Games",
-					"Electronics",
-					"Computers",
-					"Smart Home",
-					"Home",
-					"Garden & Tools",
-					"Pet Supplies",
-					"Food & Grocery",
-					"Beauty",
-					"Health",
-					"Toys",
-					"Kids",
-					"Baby",
-					"Handmade",
-					"Sports",
-					"Outdoors",
-					"Automotive",
-					"Industrial",
-					"Entertainment",
-					"Art"
+					"Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden & Tools", "Pet Supplies", "Food & Grocery", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art"
 				],
 				"demand": ["high", "medium", "medium", "low"],
 				"supply": ["high", "medium", "medium", "low"],
@@ -185,6 +183,20 @@ const config = {
 				"reviews": weightedRange(0, 35)
 			}
 
+		},
+		{
+			key: "video_id",
+			entries: 50000,
+			attributes: {
+				isFlagged: [true, false, false, false, false],
+				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
+				uploader_id: chance.guid.bind(chance),
+				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
+				rating: weightedRange(1, 5),
+				thumbs: weightedRange(0, 35),
+				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
+			}
+
 		}
 	],
 

package/models/deepNest.js

@@ -18,17 +18,17 @@ const config = {
 	groupProperties: {}
 };
 
-let formats = ['2160p', '1440p', '1080p', '720p', '480p', '360p', '240p'];
-let ratios = ['4:3', '16:10', '16:9'];
-let containers = ["WEBM", ["MPG", "MP2", "MPEG"], ["MP4", "M4P", "M4V"], ["AVI", "WMV"], ["MOV", "QT"], ["FLV", "SWF"], "AVCHD"];
-let hashtags = ["#AK", "#bitcoin", "#cureForMiley", "#faceValue", "#blm", "#fwiw", "#inappropriateFuneralSongs", "#jurassicPork", "#lolCats", "#wheatForSheep", "#momTexts", "#myWeirdGymStory", "#poppy", "#resist", "#tbt", "#wilson", "#worstGiftEver", "#yolo", "#phish", "#crypto", "#memes", "#wrongMovie", "#careerEndingTwitterTypos", "#twoThingsThatDontMix"];
-let platforms = ["Web", "Mobile Web", "Native (Android)", "Native (iOS)", "Native (Desktop)", "IoT"];
-let plans = ['free', 'premium', 'casual', 'influencer'];
-let categories = ["Product reviews video", "How-to videos", "Vlogs", "Gaming videos", "Comedy/skit videos", "Haul videos", "Memes/tags", "Favorites/best of", "Educational videos", "Unboxing videos", "Q&A videos", "Collection", "Prank videos"];
-let marketingChannels = ["Organic", "Organic", "Organic", "Organic", "Instagram Ads", "Facebook Ads", "Google Ads", "Youtube Ads", "Instagram Post", "Instagram Post", "Facebook Post"];
-
 
 function generateVideoMeta() {
+	let formats = ['2160p', '1440p', '1080p', '720p', '480p', '360p', '240p'];
+	let ratios = ['4:3', '16:10', '16:9'];
+	let containers = ["WEBM", ["MPG", "MP2", "MPEG"], ["MP4", "M4P", "M4V"], ["AVI", "WMV"], ["MOV", "QT"], ["FLV", "SWF"], "AVCHD"];
+	let hashtags = ["#AK", "#bitcoin", "#cureForMiley", "#faceValue", "#blm", "#fwiw", "#inappropriateFuneralSongs", "#jurassicPork", "#lolCats", "#wheatForSheep", "#momTexts", "#myWeirdGymStory", "#poppy", "#resist", "#tbt", "#wilson", "#worstGiftEver", "#yolo", "#phish", "#crypto", "#memes", "#wrongMovie", "#careerEndingTwitterTypos", "#twoThingsThatDontMix"];
+	let platforms = ["Web", "Mobile Web", "Native (Android)", "Native (iOS)", "Native (Desktop)", "IoT"];
+	let plans = ['free', 'premium', 'casual', 'influencer'];
+	let categories = ["Product reviews video", "How-to videos", "Vlogs", "Gaming videos", "Comedy/skit videos", "Haul videos", "Memes/tags", "Favorites/best of", "Educational videos", "Unboxing videos", "Q&A videos", "Collection", "Prank videos"];
+	let marketingChannels = ["Organic", "Organic", "Organic", "Organic", "Instagram Ads", "Facebook Ads", "Google Ads", "Youtube Ads", "Instagram Post", "Instagram Post", "Facebook Post"];
+
 
 	let videoTemplate = {
 		videoFormatInfo: {

package/models/simple.js

@@ -1,3 +1,13 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
 const Chance = require('chance');
 const chance = new Chance();
 const dayjs = require("dayjs");
@@ -24,9 +34,9 @@ const config = {
 
 	events: [
 		{
-			"event": "checkout",
-			"weight": 2,
-			"properties": {
+			event: "checkout",
+			weight: 2,
+			properties: {
 				amount: weightedRange(5, 500, 1000, .25),
 				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
 				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
@@ -35,9 +45,9 @@ const config = {
 			}
 		},
 		{
-			"event": "add to cart",
-			"weight": 4,
-			"properties": {
+			event: "add to cart",
+			weight: 4,
+			properties: {
 				amount: weightedRange(5, 500, 1000, .25),
 				rating: weightedRange(1, 5),
 				reviews: weightedRange(0, 35),
@@ -48,17 +58,17 @@ const config = {
 			}
 		},
 		{
-			"event": "page view",
-			"weight": 10,
-			"properties": {
+			event: "page view",
+			weight: 10,
+			properties: {
 				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
 				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
 			}
 		},
 		{
-			"event": "watch video",
-			"weight": 8,
-			"properties": {
+			event: "watch video",
+			weight: 8,
+			properties: {
 				videoCategory: weighList(videoCategories, integer(0, 9)),
 				isFeaturedItem: [true, false, false],
 				watchTimeSec: weightedRange(10, 600, 1000, .25),
@@ -69,9 +79,9 @@ const config = {
 			}
 		},
 		{
-			"event": "view item",
-			"weight": 8,
-			"properties": {
+			event: "view item",
+			weight: 8,
+			properties: {
 				isFeaturedItem: [true, false, false],
 				itemCategory: weighList(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
@@ -79,9 +89,9 @@ const config = {
 			}
 		},
 		{
-			"event": "save item",
-			"weight": 5,
-			"properties": {
+			event: "save item",
+			weight: 5,
+			properties: {
 				isFeaturedItem: [true, false, false],
 				itemCategory: weighList(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
@@ -89,10 +99,10 @@ const config = {
 			}
 		},
 		{
-			"event": "sign up",
-			"isFirstEvent": true,
-			"weight": 0,
-			"properties": {
+			event: "sign up",
+			isFirstEvent: true,
+			weight: 0,
+			properties: {
 				variants: ["A", "B", "C", "Control"],
 				flows: ["new", "existing", "loyal", "churned"],
 				flags: ["on", "off"],
@@ -118,6 +128,15 @@ const config = {
 	},
 
 	scdProps: {},
+	mirrorProps: {
+		isBot: { events: "*", values: [false, false, false, false, true] },
+		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
+		watchTimeSec: {
+			events: ["watch video"],
+			values: weightedRange(50, 1200, 247, 6)
+		}
+
+	},
 
 	/*
 	for group analytics keys, we need an array of arrays [[],[],[]] 

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "make-mp-data",
-  "version": "1.2.25",
+  "version": "1.3.0",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",
   "scripts": {
     "start": "node index.js",
     "dev": "./scripts/go.sh",
-	"complex": "nodemon index.js --complex",
-	"simple": "nodemon index.js",
+    "complex": "nodemon index.js --complex --e 10000 --u 100",
+    "simple": "nodemon index.js  --simple --e 10000 --u 100",
     "prune": "rm ./data/*",
     "post": "npm publish",
-    "test": "jest --runInBand",
+    "test": "NODE_ENV=test jest --runInBand",
     "deps": "sh ./scripts/deps.sh"
   },
   "repository": {
@@ -40,7 +40,7 @@
   },
   "homepage": "https://github.com/ak--47/make-mp-data#readme",
   "dependencies": {
-    "ak-tools": "^1.0.57",
+    "ak-tools": "^1.0.58",
     "chance": "^1.1.11",
     "dayjs": "^1.11.11",
     "mixpanel-import": "^2.5.51",

package/tests/e2e.test.js

@@ -24,7 +24,7 @@ describe('module', () => {
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
 		expect(lookupTableData.length).toBe(0);
-		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(scdTableData.length).toBe(0);
 		expect(userProfilesData.length).toBe(100);		
 
 	}, timeout);
@@ -46,10 +46,10 @@ describe('module', () => {
 		const results = await generate({ ...complex, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
-		expect(groupProfilesData[0]?.data?.length).toBe(350);
-		expect(lookupTableData.length).toBe(1);
+		expect(groupProfilesData[0]?.data?.length).toBe(500);
+		expect(lookupTableData.length).toBe(2);
 		expect(lookupTableData[0].data.length).toBe(1000);
-		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(scdTableData.length).toBe(5);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
@@ -61,21 +61,30 @@ describe('module', () => {
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
 		expect(lookupTableData.length).toBe(0);
-		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(scdTableData.length).toBe(0);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
 
+	test('fails with invalid configuration', async () => {
+		try {
+		  await generate({ numUsers: -10 });
+		} catch (e) {
+		  expect(e).toBeDefined();
+		}
+	  }, timeout);
+	  
+
 
 });
 
 describe('cli', () => {
 	test('works as CLI (complex)', async () => {
 		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`);
-		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: 'ignore' });
+		// expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(5);
+		expect(csvs.length).toBe(12);
 		clearData();
 	}, timeout);
 
@@ -84,7 +93,7 @@ describe('cli', () => {
 		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
 		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(2);
+		expect(csvs.length).toBe(3);
 		clearData();
 	}, timeout);
 
@@ -93,12 +102,13 @@ describe('cli', () => {
 		const run = execSync(`node ./index.js ./models/deepNest.js`);
 		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(3);
+		expect(csvs.length).toBe(2);
 		clearData();
 	}, timeout);
 
 });
 
+
 describe('options + tweaks', () => {
 	test('creates sessionIds', async () => {
 		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, sessionIds: true });

package/tests/unit.test.js

@@ -60,7 +60,7 @@ describe('utils', () => {
 		const generatedPerson = person();
 		expect(generatedPerson).toHaveProperty('$name');
 		expect(generatedPerson).toHaveProperty('$email');
-		expect(generatedPerson).toHaveProperty('$avatar');		
+		expect(generatedPerson).toHaveProperty('$avatar');
 	});
 
 
@@ -96,6 +96,38 @@ describe('utils', () => {
 		expect(result).toBe('test');
 	});
 
+	test('choose: non-function / non-array', () => {
+		expect(choose('test')).toBe('test');
+		expect(choose(123)).toBe(123);
+	});
+
+	test('choose: nested functions', () => {
+		const result = choose(() => () => () => 'nested');
+		expect(result).toBe('nested');
+	});
+
+	test('weightedRange:  within range', () => {
+		const values = weightedRange(5, 15, 100);
+		expect(values.every(v => v >= 5 && v <= 15)).toBe(true);
+		expect(values.length).toBe(100);
+	});
+
+	test('applySkew: skews', () => {
+		const value = boxMullerRandom();
+		const skewedValue = applySkew(value, .25);
+		expect(Math.abs(skewedValue)).toBeGreaterThanOrEqual(Math.abs(value));
+	});
+
+	test('mapToRange: works', () => {
+		const value = 0;
+		const mean = 10;
+		const sd = 5;
+		const mappedValue = mapToRange(value, mean, sd);
+		expect(mappedValue).toBe(10);
+	  });
+	  
+
+
 	test('exhaust: elements', () => {
 		const arr = [1, 2, 3];
 		const exhaustFn = exhaust([...arr]);
@@ -161,7 +193,7 @@ describe('utils', () => {
 		expect(typeof emojis).toBe('string');
 		const emojiArray = emojis.split(', ');
 		expect(emojiArray.length).toBeLessThanOrEqual(10); // Assuming max default is 10
-		
+
 	});
 
 

package/types.d.ts

@@ -1,82 +1,114 @@
 declare namespace main {
-  type primitives = string | number | boolean | Date | Object;
-  type valueValid =
-    | primitives
-    | primitives[]
-    | (() => primitives | primitives[]);
-
-  export interface Config {
-    token?: string;
-    seed?: string;
-    numDays?: number;
-    numEvents?: number;
-    numUsers?: number;
-    format?: "csv" | "json";
-    region?: string;
-    events?: EventConfig[];
-    superProps?: Record<string, valueValid>;
-    userProps?: Record<string, valueValid>;
-    scdProps?: Record<string, valueValid>;
-    groupKeys?: [string, number][];
-    groupProps?: Record<string, GroupProperty>; // Adjust according to usage
-    lookupTables?: LookupTable[];
-    writeToDisk?: boolean;
-    simulationName?: string;
-    verbose?: boolean;
-    anonIds?: boolean;
-    sessionIds?: boolean;
-    hook?: Hook;
+	type Primitives = string | number | boolean | Date | Record<string, any>;
+  
+	// Recursive type to handle functions returning functions that eventually return Primitives or arrays of Primitives
+	type ValueValid =
+	  | Primitives
+	  | ValueValid[]
+	  | (() => ValueValid);
+  
+	// MAIN CONFIGURATION OBJECT
+	export interface Config {
+	  token?: string;
+	  seed?: string;
+	  numDays?: number;
+	  numEvents?: number;
+	  numUsers?: number;
+	  format?: "csv" | "json";
+	  region?: string;
+	  events?: EventConfig[];
+	  superProps?: Record<string, ValueValid>;
+	  userProps?: Record<string, ValueValid>;
+	  scdProps?: Record<string, ValueValid>;
+	  mirrorProps?: Record<string, MirrorProps>;
+	  groupKeys?: [string, number][];
+	  groupProps?: Record<string, Record<string, ValueValid>>;
+	  lookupTables?: LookupTable[];
+	  writeToDisk?: boolean;
+	  simulationName?: string;
+	  verbose?: boolean;
+	  anonIds?: boolean;
+	  sessionIds?: boolean;
+	  hook?: Hook;
+	}
+  
+	export type Hook = (record: any, type: string, meta: any) => any;
+  
+	export interface EventConfig {
+	  event?: string;
+	  weight?: number;
+	  properties?: Record<string, ValueValid>;
+	  isFirstEvent?: boolean;
+	}
+  
+	export interface MirrorProps {
+	  events: string[] | "*";
+	  values: ValueValid[];
+	}
+  
+	export interface LookupTable {
+	  key: string;
+	  entries: number;
+	  attributes: Record<string, ValueValid>;
+	}
+  
+	export interface SCDTable {
+	  distinct_id: string;
+	  insertTime: string;
+	  startTime: string;
+	  [key: string]: ValueValid;
+	}
+  
+	export type Result = {
+	  eventData: EventData[];
+	  userProfilesData: any[];
+	  scdTableData: any[];
+	  groupProfilesData: GroupProfilesData[];
+	  lookupTableData: LookupTableData[];
+	  import?: ImportResults;
+	  files?: string[];
+	};
+  
+	export interface EventData {
+	  event: string;
+	  $source: string;
+	  time: string;
+	  $device_id?: string;
+	  $session_id?: string;
+	  $user_id?: string;
+	  [key: string]: any;
+	}
+  
+	export interface GroupProfilesData {
+	  key: string;
+	  data: any[];
+	}
+  
+	export interface LookupTableData {
+	  key: string;
+	  data: any[];
+	}
+  
+	export interface ImportResults {
+	  events: ImportResult;
+	  users: ImportResult;
+	  groups: ImportResult[];
+	}
+  
+	export interface ImportResult {
+	  success: number;
+	  bytes: number;
+	}
   }
-
-  export type Hook = (record: any, type: string, meta: any) => any;
-
-  interface EventConfig {
-    event?: string;
-    weight?: number;
-    properties?: {
-      [key: string]: valueValid; // Consider refining based on actual properties used
-    };
-    isFirstEvent?: boolean;
-  }
-
-  interface GroupProperty {
-    [key?: string]: valueValid;
-  }
-
-  interface LookupTable {
-    key: string;
-    entries: number;
-    attributes: {
-      [key?: string]: valueValid;
-    };
-  }
-
-  type Result = {
-    eventData: {
-      event: any;
-      $source: string;
-    }[];
-    userProfilesData: any[];
-    scdTableData: any[];
-    groupProfilesData: {
-      key: string;
-      data: any[];
-    }[];
-    lookupTableData: {
-      key: string;
-      data: any[];
-    }[];
-    import?: undefined;
-    files?: undefined;
-  };
-}
-
-/**
- * Mixpanel Data Generator
- * model events, users, groups, and lookup tables (and SCD props!)
- * @example
- * const gen = require('make-mp-data')
- * const dta = gen({writeToDisk: false})
- */
-declare function main(config: main.Config): Promise<main.Result>;
-export = main;
+  
+  /**
+   * Mixpanel Data Generator
+   * model events, users, groups, and lookup tables (and SCD props!)
+   * @example
+   * const gen = require('make-mp-data')
+   * const dta = gen({writeToDisk: false})
+   */
+  declare function main(config: main.Config): Promise<main.Result>;
+  
+  export = main;
+  

package/utils.js

@@ -155,7 +155,7 @@ function mapToRange(value, mean, sd) {
 	return Math.round(value * sd + mean);
 };
 
-function weightedRange(min, max, size = 100, skew = 1) {
+function unOptimizedWeightedRange(min, max, size = 100, skew = 1) {
 	const mean = (max + min) / 2;
 	const sd = (max - min) / 4;
 	let array = [];
@@ -176,6 +176,23 @@ function weightedRange(min, max, size = 100, skew = 1) {
 	return array;
 };
 
+// optimized weighted range
+function weightedRange(min, max, size = 100, skew = 1) {
+    const mean = (max + min) / 2;
+    const sd = (max - min) / 4;
+    const array = [];
+    while (array.length < size) {
+        const normalValue = boxMullerRandom();
+        const skewedValue = applySkew(normalValue, skew);
+        const mappedValue = mapToRange(skewedValue, mean, sd);
+        if (mappedValue >= min && mappedValue <= max) {
+            array.push(mappedValue);
+        }
+    }
+    return array;
+}
+
+
 function progress(thing, p) {
 	readline.cursorTo(process.stdout, 0);
 	process.stdout.write(`${thing} processed ... ${comma(p)}`);