make-mp-data 1.2.21 → 1.3.0

package/.vscode/launch.json

@@ -4,6 +4,18 @@
 	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
 	"version": "0.2.0",
 	"configurations": [
+		{
+			"command": "npm run simple",
+			"name": "simple",
+			"request": "launch",
+			"type": "node-terminal"
+		},
+		{
+			"command": "npm run complex",
+			"name": "complex",
+			"request": "launch",
+			"type": "node-terminal"
+		},			
 		{
 			"type": "node",
 			"request": "launch",
@@ -15,5 +27,6 @@
 			"internalConsoleOptions": "neverOpen",
 			"args": ["--ignore", "./data/"]
 		}
+
 	]
 }

package/.vscode/settings.json

@@ -1,7 +1,25 @@
 {
 	"cSpell.words": [
+		"AVCHD",
+		"chupacabra",
+		"darr",
+		"dislikers",
+		"Dont",
+		"jackalope",
+		"mbembe",
+		"megalodon",
+		"Miley",
+		"mokele",
+		"mothman",
+		"nessie",
+		"nesty",
+		"planthopper",
+		"psyllid",
+		"tatzelwurm",
 		"timesoup",
-		"unparse"
+		"unparse",
+		"Vlogs",
+		"weindgo"
 	],
 	"jest.runMode": "on-demand",
 	"jest.jestCommandLine": "npm run test --",

package/README.md

@@ -1,50 +1,103 @@
 
-
 # Make Mixpanel Data
-a quick and dirty CLI in node.js to generate fake data for mixpanel.
 
-## tldr;
+## 🤖 What is this?
+
+Generate fake Mixpanel data _quickly_ and _easily_ with a simple CLI in Node.js. **Create events, user profiles, groups profiles, SCD data, mirror tables, and lookup tables**; basically everything you need for testing + demoing mixpanel analysis features. 
+
+the CLI looks like this:
+
+![Generate Mixpanel data](https://aktunes.neocities.org/makeDataDemo.gif)
+
+under the hood, `make-mp-data` is modeling data adherent to match [Mixpanel's data model](https://docs.mixpanel.com/docs/data-structure/concepts), giving you the tools you need for robust, realistical field ready test data.
+
+## 🚀 Quick Start
+
+### Basic Usage
+
+Generate events and users, and write them to CSV files:
 
- 
- ```bash
+```bash
 npx make-mp-data 
 ```
- - makes events + users (and writes them to CSVs)
 
- ```bash
-npx make-mp-data --numUsers 100 --numEvents 10000 --days 90 --format json
+### Customizing Output
+
+Generate 10,000 events and 100 users over the last 90 days, and write the output as JSON:
+
+```bash
+npx make-mp-data --numUsers 100 --numEvents 10000 --numDays 90 --format json
 ```
- - makes ~10k events + 100 users from the last 90 days (but writes JSON)
 
+### Complex Data Models
 
- ```bash
+Create a comprehensive dataset including events, users, groups, SCD, and lookup tables:
+
+```bash
 npx make-mp-data --complex
 ```
- - makes events + users + groups + scd + lookup tables
- - this includes every type of data that mixpanel supports
+
+### Send Data to Mixpanel
+
+Generate and send data directly to Mixpanel using your project token:
 
 ```bash
 npx make-mp-data --token 1234
 ```
- - makes events + users (and send them to mixpanel)
+
+### Help and Options
+
+Need more info on available options?
 
 ```bash
 npx make-mp-data --help
 ```
-- explains all the options you can specify
 
-## customization
+## 🔧 Schema
+
+To choose your own event and property names and values, specify a custom data model and additional options:
 
 ```bash
 npx make-mp-data [dataModel.js] [options]
 ```
-ex.
+
+Example:
 
 ```bash
 npx make-mp-data ecommSpec.js --token 1234 --numDays 30 --numUsers 1000 --numEvents 1000000
 ```
 
-see `--help` for a full list of options
+### Data Models
+
+Check out `./models/` for example `dataModel.js` files to get started quickly.
+
+## 🛠️ CLI Options
+
+Here's a breakdown of the CLI options you can use with `make-mp-data`:
+
+- `--numUsers`: Number of users to generate.
+- `--numEvents`: Number of events to generate.
+- `--numDays`: Number of days over which to spread the generated data.
+- `--format`: Output format (`csv` or `json`).
+- `--token`: Mixpanel project token for direct data import.
+- `--region`: Mixpanel data region (`US`, `EU`).
+- `--writeToDisk`: Whether to write the data to disk (`true` or `false`).
+- `--verbose`: Enable verbose logging.
+- `--complex`: create a complex set models including groups, SCD, and lookup tables.
+- `--simple`: create a simple dataset including events, and users
+
+## 📄 Examples
+
+Check out the examples directory for sample data models:
+
+```bash
+ls ./models/
+```
+
+These models provide a great starting point for creating your own custom data generation scripts.
+
+## 🤝 Contributing
 
-see `./models/` for a few `dataModel.js` examples...
+Feel free to fork this repository and submit pull requests. Contributions are always welcome!
 
+For any issues or feature requests, please create an issue on the [GitHub repository](https://github.com/ak--47/make-mp-data/issues).

package/index.js

@@ -1,6 +1,5 @@
 #! /usr/bin/env node
 
-
 /*
 make fake mixpanel data easily!
 by AK 
@@ -20,9 +19,16 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
-const { makeName, md5 } = require('ak-tools');
+const { makeName, md5, clone, tracker, uid } = require('ak-tools');
 const NOW = dayjs().unix();
 let VERBOSE = false;
+let isCLI = false;
+
+const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41");
+function track() {
+	if (process.env.NODE_ENV === 'test') return;
+	metrics.track(...arguments);
+}
 
 /** @typedef {import('./types.d.ts').Config} Config */
 /** @typedef {import('./types.d.ts').EventConfig} EventConfig */
@@ -41,26 +47,45 @@ async function main(config) {
 		superProps = { platform: ["web", "iOS", "Android"] },
 		userProps = {
 			favoriteColor: ["red", "green", "blue", "yellow"],
-			spiritAnimal: chance.animal,
+			spiritAnimal: chance.animal.bind(chance),
 		},
-		scdProps = { NPS: u.weightedRange(0, 10, 150, 1.6) },
+		scdProps = {},
+		mirrorProps = {},
 		groupKeys = [],
 		groupProps = {},
 		lookupTables = [],
-		anonIds = true,
-		sessionIds = true,
+		anonIds = false,
+		sessionIds = false,
 		format = "csv",
 		token = null,
 		region = "US",
 		writeToDisk = false,
 		verbose = false,
+		hook = (record) => record,
 	} = config;
 	VERBOSE = verbose;
 	config.simulationName = makeName();
+	const { simulationName } = config;
 	global.MP_SIMULATION_CONFIG = config;
 	const uuidChance = new Chance(seed);
+	const runId = uid(42);
+	track('start simulation', {
+		runId,
+		seed,
+		numEvents,
+		numUsers,
+		numDays,
+		events,
+		anonIds,
+		sessionIds,
+		format,
+		token,
+		region,
+		writeToDisk,
+		isCLI
+	});
 	log(`------------------SETUP------------------`);
-	log(`\nyour data simulation will heretofore be known as: \n\n\t${config.simulationName.toUpperCase()}...\n`);
+	log(`\nyour data simulation will heretofore be known as: \n\n\t${simulationName.toUpperCase()}...\n`);
 	log(`and your configuration is:\n\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk }, null, 2));
 	log(`------------------SETUP------------------`, "\n");
 
@@ -97,11 +122,16 @@ async function main(config) {
 		.filter((e) => !e.isFirstEvent);
 
 	const firstEvents = events.filter((e) => e.isFirstEvent);
-	const eventData = [];
-	const userProfilesData = [];
-	let scdTableData = [];
-	const groupProfilesData = [];
-	const lookupTableData = [];
+	const eventData = enrichArray([], { hook, type: "event", config });
+	const userProfilesData = enrichArray([], { hook, type: "user", config });
+	const scdTableKeys = Object.keys(scdProps);
+	const scdTableData = [];
+	for (const [index, key] of scdTableKeys.entries()) {
+		scdTableData[index] = enrichArray([], { hook, type: "scd", config, scdKey: key });
+	}
+	// const scdTableData = enrichArray([], { hook, type: "scd", config });
+	const groupProfilesData = enrichArray([], { hook, type: "groups", config });
+	const lookupTableData = enrichArray([], { hook, type: "lookups", config });
 	const avgEvPerUser = Math.floor(numEvents / numUsers);
 
 	//user loop
@@ -110,15 +140,20 @@ async function main(config) {
 		u.progress("users", i);
 		const user = generateUser();
 		const { distinct_id, $created, anonymousIds, sessionIds } = user;
-		userProfilesData.push(makeProfile(userProps, user));
-		const mutations = chance.integer({ min: 1, max: 10 });
-		scdTableData.push(makeSCD(scdProps, distinct_id, mutations, $created));
+		userProfilesData.hPush(makeProfile(userProps, user));
+
+		//scd loop
+		for (const [index, key] of scdTableKeys.entries()) {
+			const mutations = chance.integer({ min: 1, max: 10 });
+			scdTableData[index].hPush(makeSCD(scdProps[key], key, distinct_id, mutations, $created));
+		}
+
 		const numEventsThisUser = Math.round(
 			chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / u.integer(3, 7) })
 		);
-		
+
 		if (firstEvents.length) {
-			eventData.push(
+			eventData.hPush(
 				makeEvent(
 					distinct_id,
 					anonymousIds,
@@ -134,7 +169,7 @@ async function main(config) {
 
 		//event loop
 		for (let j = 0; j < numEventsThisUser; j++) {
-			eventData.push(
+			eventData.hPush(
 				makeEvent(
 					distinct_id,
 					anonymousIds,
@@ -149,7 +184,7 @@ async function main(config) {
 	}
 
 	//flatten SCD
-	scdTableData = scdTableData.flat();
+	scdTableData.forEach((table, index) => scdTableData[index] = table.flat());
 
 	log("\n");
 
@@ -163,11 +198,11 @@ async function main(config) {
 			const group = {
 				[groupKey]: i,
 				...makeProfile(groupProps[groupKey]),
-				$distinct_id: i,
+				// $distinct_id: i,
 			};
 			groupProfiles.push(group);
 		}
-		groupProfilesData.push({ key: groupKey, data: groupProfiles });
+		groupProfilesData.hPush({ key: groupKey, data: groupProfiles });
 	}
 	log("\n");
 
@@ -183,27 +218,58 @@ async function main(config) {
 			};
 			data.push(item);
 		}
-		lookupTableData.push({ key, data });
+		lookupTableData.hPush({ key, data });
+	}
+
+	// deal with mirror props
+	let mirrorEventData = [];
+	const mirrorPropKeys = Object.keys(mirrorProps);
+	if (mirrorPropKeys.length) {
+		mirrorEventData = clone(eventData);
+		for (const row of mirrorEventData) {
+			for (const key of mirrorPropKeys) {
+				if (mirrorProps[key]?.events?.includes(row?.event)) row[key] = hook(u.choose(mirrorProps[key]?.values), "mirror", { config, row, key });
+				if (mirrorProps[key]?.events === "*") row[key] = hook(u.choose(mirrorProps[key]?.values), "mirror", { config, row, key });
+			}
+		}
 	}
-	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder } =
+
+	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder } =
 		buildFileNames(config);
 	const pairs = [
-		[eventFiles, eventData],
-		[userFiles, userProfilesData],
+		[eventFiles, [eventData]],
+		[userFiles, [userProfilesData]],
 		[scdFiles, scdTableData],
 		[groupFiles, groupProfilesData],
 		[lookupFiles, lookupTableData],
+		[mirrorFiles, [mirrorEventData]],
 	];
-	log("\n")
+	log("\n");
 	log(`---------------SIMULATION----------------`, "\n");
 
 	if (!writeToDisk && !token) {
+		track('end simulation', {
+			runId,
+			seed,
+			numEvents,
+			numUsers,
+			numDays,
+			events,
+			anonIds,
+			sessionIds,
+			format,
+			token,
+			region,
+			writeToDisk,
+			isCLI
+		});
 		return {
 			eventData,
 			userProfilesData,
 			scdTableData,
 			groupProfilesData,
 			lookupTableData,
+			mirrorEventData,
 			import: {},
 			files: []
 		};
@@ -211,32 +277,39 @@ async function main(config) {
 	log(`-----------------WRITES------------------`, `\n\n`);
 	//write the files
 	if (writeToDisk) {
-		if (verbose) log(`writing files... for ${config.simulationName}`);
-		loopFiles: for (const pair of pairs) {
-			const [paths, data] = pair;
+		if (verbose) log(`writing files... for ${simulationName}`);
+		loopFiles: for (const ENTITY of pairs) {
+			const [paths, data] = ENTITY;
 			if (!data.length) continue loopFiles;
-			for (const path of paths) {
-				let datasetsToWrite;
-				if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
-				else datasetsToWrite = [data];
-				for (const writeData of datasetsToWrite) {
-					//if it's a lookup table, it's always a CSV
-					if (format === "csv" || path.includes("-LOOKUP.csv")) {
-						log(`\twriting ${path}`);
-						const columns = u.getUniqueKeys(writeData);
-						//papa parse needs nested JSON stringified
-						writeData.forEach((e) => {
-							for (const key in e) {
-								if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
-							}
-						});
-						const csv = Papa.unparse(writeData, { columns });
-						await touch(path, csv);
-					} else {
-						const ndjson = data.map((d) => JSON.stringify(d)).join("\n");
-						await touch(path, ndjson, false);
-					}
+			for (const [index, path] of paths.entries()) {
+				let TABLE;
+				//group + lookup tables are structured differently
+				if (data?.[index]?.["key"]) {
+					TABLE = data[index].data;
+				}
+				else {
+					TABLE = data[index];
+				}
+
+				log(`\twriting ${path}`);
+				//if it's a lookup table, it's always a CSV
+				if (format === "csv" || path.includes("-LOOKUP.csv")) {
+					const columns = u.getUniqueKeys(TABLE);
+					//papa parse needs eac nested field JSON stringified
+					TABLE.forEach((e) => {
+						for (const key in e) {
+							if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
+						}
+					});
+
+					const csv = Papa.unparse(TABLE, { columns });
+					await touch(path, csv);
+				}
+				else {
+					const ndjson = TABLE.map((d) => JSON.stringify(d)).join("\n");
+					await touch(path, ndjson, false);
 				}
+
 			}
 		}
 	}
@@ -297,9 +370,24 @@ async function main(config) {
 
 	}
 	log(`\n-----------------WRITES------------------`, "\n");
+	track('end simulation', {
+		runId,
+		seed,
+		numEvents,
+		numUsers,
+		numDays,
+		events,
+		anonIds,
+		sessionIds,
+		format,
+		token,
+		region,
+		writeToDisk,
+		isCLI
+	});
 	return {
 		import: importResults,
-		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder],
+		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder],
 	};
 }
 
@@ -322,16 +410,23 @@ function makeProfile(props, defaults) {
 
 	return profile;
 }
-
-function makeSCD(props, distinct_id, mutations, $created) {
-	if (JSON.stringify(props) === "{}") return [];
+/**
+ * @param  {import('./types.d.ts').valueValid} prop
+ * @param  {string} scdKey
+ * @param  {string} distinct_id
+ * @param  {number} mutations
+ * @param  {string} $created
+ */
+function makeSCD(prop, scdKey, distinct_id, mutations, $created) {
+	if (JSON.stringify(prop) === "{}") return {};
+	if (JSON.stringify(prop) === "[]") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs($created);
 	const deltaDays = dayjs().diff(lastInserted, "day");
 
 	for (let i = 0; i < mutations; i++) {
 		if (lastInserted.isAfter(dayjs())) break;
-		const scd = makeProfile(props, { distinct_id });
+		const scd = makeProfile({ [scdKey]: prop }, { distinct_id });
 		scd.startTime = lastInserted.toISOString();
 		lastInserted = lastInserted.add(u.integer(1, 1000), "seconds");
 		scd.insertTime = lastInserted.toISOString();
@@ -420,12 +515,21 @@ function buildFileNames(config) {
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
-		scdFiles: [path.join(writeDir, `${simName}-SCD.${extension}`)],
+		scdFiles: [],
+		mirrorFiles: [path.join(writeDir, `${simName}-EVENTS-FUTURE-MIRROR.${extension}`)],
 		groupFiles: [],
 		lookupFiles: [],
 		folder: writeDir,
 	};
 
+	//add SCD files
+	const scdKeys = Object.keys(config?.scdProps || {});
+	for (const key of scdKeys) {
+		writePaths.scdFiles.push(
+			path.join(writeDir, `${simName}-${key}-SCD.${extension}`)
+		);
+	}
+
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
 		writePaths.groupFiles.push(
@@ -445,10 +549,23 @@ function buildFileNames(config) {
 }
 
 
+function enrichArray(arr = [], opts = {}) {
+	const { hook = a => a, type = "", ...rest } = opts;
+
+	function transformThenPush(item) {
+		return arr.push(hook(item, type, rest));
+	}
+
+	arr.hPush = transformThenPush;
+
+	return arr;
+};
+
 
 
 // this is for CLI
 if (require.main === module) {
+	isCLI = true;
 	const args = cliParams();
 	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false } = args;
 	const suppliedConfig = args._[0];
@@ -489,8 +606,8 @@ if (require.main === module) {
 	main(config)
 		.then((data) => {
 			log(`-----------------SUMMARY-----------------`);
-			const d = {success: 0, bytes: 0};
-			const darr = [d]
+			const d = { success: 0, bytes: 0 };
+			const darr = [d];
 			const { events = d, groups = darr, users = d } = data.import;
 			const files = data.files;
 			const folder = files?.pop();
@@ -530,5 +647,13 @@ if (require.main === module) {
 
 
 function log(...args) {
+	const cwd = process.cwd();  // Get the current working directory
+
+	for (let i = 0; i < args.length; i++) {
+		// Replace occurrences of the current working directory with "./" in string arguments
+		if (typeof args[i] === 'string') {
+			args[i] = args[i].replace(new RegExp(cwd, 'g'), ".");
+		}
+	}
 	if (VERBOSE) console.log(...args);
 }