make-mp-data 1.2.2 → 1.2.21

package/README.md

@@ -5,13 +5,33 @@ a quick and dirty CLI in node.js to generate fake data for mixpanel.
 
 ## tldr;
 
+ 
+ ```bash
+npx make-mp-data 
+```
+ - makes events + users (and writes them to CSVs)
+
+ ```bash
+npx make-mp-data --numUsers 100 --numEvents 10000 --days 90 --format json
+```
+ - makes ~10k events + 100 users from the last 90 days (but writes JSON)
+
+
+ ```bash
+npx make-mp-data --complex
+```
+ - makes events + users + groups + scd + lookup tables
+ - this includes every type of data that mixpanel supports
+
 ```bash
 npx make-mp-data --token 1234
 ```
- - makes events, users, groups (sends them to your project)
- - makes lookup tables and SCD type 2 exported as CSVs
+ - makes events + users (and send them to mixpanel)
 
-(note: if you want group analytics, add a `company_id` group key to your project before running)
+```bash
+npx make-mp-data --help
+```
+- explains all the options you can specify
 
 ## customization
 
@@ -26,5 +46,5 @@ npx make-mp-data ecommSpec.js --token 1234 --numDays 30 --numUsers 1000 --numEve
 
 see `--help` for a full list of options
 
-see `./models/` for a few `dataModel.js`` examples
+see `./models/` for a few `dataModel.js` examples...
 

package/index.js

@@ -61,7 +61,7 @@ async function main(config) {
 	const uuidChance = new Chance(seed);
 	log(`------------------SETUP------------------`);
 	log(`\nyour data simulation will heretofore be known as: \n\n\t${config.simulationName.toUpperCase()}...\n`);
-	log(`and your configuration is:\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk }, null, 2));
+	log(`and your configuration is:\n\n`, JSON.stringify({ seed, numEvents, numUsers, numDays, format, token, region, writeToDisk }, null, 2));
 	log(`------------------SETUP------------------`, "\n");
 
 
@@ -116,7 +116,7 @@ async function main(config) {
 		const numEventsThisUser = Math.round(
 			chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / u.integer(3, 7) })
 		);
-
+		
 		if (firstEvents.length) {
 			eventData.push(
 				makeEvent(
@@ -147,6 +147,7 @@ async function main(config) {
 			);
 		}
 	}
+
 	//flatten SCD
 	scdTableData = scdTableData.flat();
 
@@ -193,7 +194,8 @@ async function main(config) {
 		[groupFiles, groupProfilesData],
 		[lookupFiles, lookupTableData],
 	];
-	log("\n", `---------------SIMULATION----------------`, "\n");
+	log("\n")
+	log(`---------------SIMULATION----------------`, "\n");
 
 	if (!writeToDisk && !token) {
 		return {
@@ -202,12 +204,14 @@ async function main(config) {
 			scdTableData,
 			groupProfilesData,
 			lookupTableData,
+			import: {},
+			files: []
 		};
 	}
 	log(`-----------------WRITES------------------`, `\n\n`);
 	//write the files
 	if (writeToDisk) {
-		if (verbose) log(`writing files... for ${config.simulationName}\n`);
+		if (verbose) log(`writing files... for ${config.simulationName}`);
 		loopFiles: for (const pair of pairs) {
 			const [paths, data] = pair;
 			if (!data.length) continue loopFiles;
@@ -218,7 +222,7 @@ async function main(config) {
 				for (const writeData of datasetsToWrite) {
 					//if it's a lookup table, it's always a CSV
 					if (format === "csv" || path.includes("-LOOKUP.csv")) {
-						log(`writing ${path}`);
+						log(`\twriting ${path}`);
 						const columns = u.getUniqueKeys(writeData);
 						//papa parse needs nested JSON stringified
 						writeData.forEach((e) => {
@@ -228,7 +232,6 @@ async function main(config) {
 						});
 						const csv = Papa.unparse(writeData, { columns });
 						await touch(path, csv);
-						log(`\tdone\n`);
 					} else {
 						const ndjson = data.map((d) => JSON.stringify(d)).join("\n");
 						await touch(path, ndjson, false);
@@ -293,7 +296,7 @@ async function main(config) {
 		}
 
 	}
-	log(`-----------------WRITES------------------`, "\n");
+	log(`\n-----------------WRITES------------------`, "\n");
 	return {
 		import: importResults,
 		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder],
@@ -311,7 +314,7 @@ function makeProfile(props, defaults) {
 
 	for (const key in props) {
 		try {
-			profile[key] = choose(props[key]);
+			profile[key] = u.choose(props[key]);
 		} catch (e) {
 			// debugger;
 		}
@@ -451,20 +454,22 @@ if (require.main === module) {
 	const suppliedConfig = args._[0];
 
 	//if the user specifics an separate config file
+	//todo this text isn't displaying
 	let config = null;
 	if (suppliedConfig) {
-		log(`using ${suppliedConfig} for data\n`);
+		console.log(`using ${suppliedConfig} for data\n`);
 		config = require(path.resolve(suppliedConfig));
 	}
 	else {
 		if (complex) {
-			log(`... using default COMPLEX configuration [everything] ...\n`);
-			log(`... for more simple data, don't use the --complex flag ...\n`);
+			console.log(`... using default COMPLEX configuration [everything] ...\n`);
+			console.log(`... for more simple data, don't use the --complex flag ...\n`);
+			console.log(`... or specify your own js config file (see docs or --help) ...\n`);
 			config = require(path.resolve(__dirname, "./models/complex.js"));
 		}
 		else {
-			log(`... using default SIMPLE configuration [events + users] ...\n`);
-			log(`... for more complex data, use the --complex flag ...\n`);
+			console.log(`... using default SIMPLE configuration [events + users] ...\n`);
+			console.log(`... for more complex data, use the --complex flag ...\n`);
 			config = require(path.resolve(__dirname, "./models/simple.js"));
 		}
 	}
@@ -484,7 +489,9 @@ if (require.main === module) {
 	main(config)
 		.then((data) => {
 			log(`-----------------SUMMARY-----------------`);
-			const { events, groups, users } = data.import;
+			const d = {success: 0, bytes: 0};
+			const darr = [d]
+			const { events = d, groups = darr, users = d } = data.import;
 			const files = data.files;
 			const folder = files?.pop();
 			const groupBytes = groups.reduce((acc, group) => {
@@ -501,7 +508,7 @@ if (require.main === module) {
 				bytes: bytesHuman(bytes || 0),
 			};
 			if (bytes > 0) console.table(stats);
-			log(`\nfiles written to ${folder} ...`);
+			log(`\nfiles written to ${folder || "no where; we didn't write anything"} ...`);
 			log("  " + files?.flat().join("\n  "));
 			log(`\n----------------SUMMARY-----------------\n\n\n`);
 		})

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "make-mp-data",
-  "version": "1.2.02",
+  "version": "1.2.21",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",
   "scripts": {
     "start": "node index.js",
-    "prune": "rm ./data/*",
-    "go": "sh ./scripts/go.sh",
+	"dev": "./scripts/go.sh",
+    "prune": "rm ./data/*",    
     "post": "npm publish",
     "test": "jest --runInBand",
     "deps": "sh ./scripts/deps.sh"

package/scripts/go.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-RUNTIME=local node --inspect index.js --writeToDisk false -u 100 -e 10000
+RUNTIME=local nodemon --inspect index.js --writeToDisk false -u 100 -e 10000

package/tests/unit.test.js

@@ -70,9 +70,7 @@ describe('utils', () => {
 		const generatedPerson = person();
 		expect(generatedPerson).toHaveProperty('$name');
 		expect(generatedPerson).toHaveProperty('$email');
-		expect(generatedPerson).toHaveProperty('$avatar');
-		expect(generatedPerson).toHaveProperty('anonymousIds');
-		expect(generatedPerson.anonymousIds).toBeInstanceOf(Array);
+		expect(generatedPerson).toHaveProperty('$avatar');		
 	});
 
 

package/utils.js

@@ -81,16 +81,24 @@ function day(start, end) {
 };
 
 function choose(value) {
-	if (typeof value === 'function') {
-		return value();
-	}
-	if (Array.isArray(value)) {
-		return chance.pickone(value);
-	}
+	try {
+		// Keep resolving the value if it's a function
+		while (typeof value === 'function') {
+			value = value();
+		}
 
-	return value;
-};
+		// Now, if the resolved value is an array, use chance.pickone
+		if (Array.isArray(value)) {
+			return chance.pickone(value);
+		}
 
+		// If it's not a function or array, return it as is
+		return value;
+	}
+	catch (e) {
+		return '';
+	}
+}
 function exhaust(arr) {
 	return function () {
 		return arr.shift();
@@ -285,30 +293,34 @@ function person(bornDaysAgo = 30) {
 	const $avatar = avatarPrefix + avPath;
 	const $created = date(bornDaysAgo, true, null)();
 
+	const user = {
+		$name,
+		$email,
+		$avatar,
+		$created,
+	};
+
 	//anon Ids
-	const anonymousIds = [];
-	const clusterSize = integer(2, 10);
-	for (let i = 0; i < clusterSize; i++) {
-		anonymousIds.push(uid(42));
+	if (global.MP_SIMULATION_CONFIG?.anonIds) {
+		const anonymousIds = [];
+		const clusterSize = integer(2, 10);
+		for (let i = 0; i < clusterSize; i++) {
+			anonymousIds.push(uid(42));
+		}
+		user.anonymousIds = anonymousIds;
 	}
 
 	//session Ids
-	const sessionIds = [];
-	const sessionSize = integer(5, 30);
-	for (let i = 0; i < sessionSize; i++) {
-		sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
+	if (global.MP_SIMULATION_CONFIG?.sessionIds) {
+		const sessionIds = [];
+		const sessionSize = integer(5, 30);
+		for (let i = 0; i < sessionSize; i++) {
+			sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
+		}
+		user.sessionIds = sessionIds;
 	}
 
-
-
-	return {
-		$name,
-		$email,
-		$avatar,
-		$created,
-		anonymousIds,
-		sessionIds
-	};
+	return user;
 };