make-mp-data 1.0.4 → 1.0.6

package/default.js

@@ -94,7 +94,7 @@ const config = {
 	each pair represents a group_key and the number of profiles for that key
 	*/
 	groupKeys: [
-		['company_id', 250],
+		['company_id', 350],
 
 	],
 	groupProps: {

package/index.js

@@ -87,6 +87,7 @@ async function main(config) {
 		format = "csv",
 		token = null,
 		region = "US",
+		writeToDisk = true
 	} = config;
 
 	const uuidChance = new Chance(seed);
@@ -115,7 +116,7 @@ async function main(config) {
 	const firstEvents = events.filter((e) => e.isFirstEvent);
 	const eventData = [];
 	const userProfilesData = [];
-	const scdTableData = [];
+	let scdTableData = [];
 	const groupProfilesData = [];
 	const lookupTableData = [];
 	const avgEvPerUser = Math.floor(numEvents / numUsers);
@@ -158,6 +159,9 @@ async function main(config) {
 			);
 		}
 	}
+	//flatten SCD
+	scdTableData = scdTableData.flat();
+
 	console.log("\n");
 
 	// make group profiles
@@ -201,17 +205,24 @@ async function main(config) {
 		[lookupFiles, lookupTableData],
 	];
 	console.log("\n");
+
+	if (!writeToDisk) return { eventData, userProfilesData, scdTableData, groupProfilesData, lookupTableData };
 	//write the files
 	for (const pair of pairs) {
 		const [paths, data] = pair;
 		for (const path of paths) {
-			if (format === "csv") {
-				console.log(`writing ${path}`);
-				const csv = Papa.unparse(data, {});
-				await touch(path, csv);
-				console.log(`\tdone\n`);
-			} else {
-				await touch(path, data, true);
+			let datasetsToWrite;
+			if (data?.[0]?.["key"]) datasetsToWrite = data.map(d => d.data);
+			else datasetsToWrite = [data];
+			for (const writeData of datasetsToWrite) {
+				if (format === "csv") {
+					console.log(`writing ${path}`);
+					const csv = Papa.unparse(writeData, {});
+					await touch(path, csv);
+					console.log(`\tdone\n`);
+				} else {
+					await touch(path, data, true);
+				}
 			}
 		}
 	}
@@ -363,34 +374,45 @@ function buildFileNames(config) {
 	return writePaths;
 }
 
-//that's all folks :)
-main(config)
-	.then((data) => {
-		console.log(`------------------SUMMARY------------------`);
-		const { events, groups, users } = data.import;
-		const files = data.files;
-		const folder = files.pop();
-		const groupBytes = groups.reduce((acc, group) => { return acc + group.bytes; }, 0);
-		const groupSuccess = groups.reduce((acc, group) => { return acc + group.success; }, 0);
-		const bytes = events.bytes + groupBytes + users.bytes;
-		const stats = {
-			events: comma(events.success || 0),
-			users: comma(users.success || 0),
-			groups: comma(groupSuccess || 0),
-			bytes: bytesHuman(bytes || 0)
-		};
-		console.table(stats);
-		console.log(`\nfiles written to ${folder}...`);
-		console.log("\t" + files.flat().join('\t\n'));
-		console.log(`------------------SUMMARY------------------\n\n\n`);
-	})
-	.catch((e) => {
-		console.log(`------------------ERROR------------------`);
-		console.error(e);
-		console.log(`------------------ERROR------------------`);
-		debugger;
-	})
-	.finally(() => {
-		console.log('have a wonderful day :)');
-		openFinder(path.resolve("./data"));
-	});
+// this is for CLI
+if (require.main === module) {
+	//that's all folks :)
+	main(config)
+		.then((data) => {
+			console.log(`------------------SUMMARY------------------`);
+			const { events, groups, users } = data.import;
+			const files = data.files;
+			const folder = files.pop();
+			const groupBytes = groups.reduce((acc, group) => { return acc + group.bytes; }, 0);
+			const groupSuccess = groups.reduce((acc, group) => { return acc + group.success; }, 0);
+			const bytes = events.bytes + groupBytes + users.bytes;
+			const stats = {
+				events: comma(events.success || 0),
+				users: comma(users.success || 0),
+				groups: comma(groupSuccess || 0),
+				bytes: bytesHuman(bytes || 0)
+			};
+			if (bytes > 0) console.table(stats);
+			console.log(`\nfiles written to ${folder}...`);
+			console.log("\t" + files.flat().join('\n\t'));
+			console.log(`\n------------------SUMMARY------------------\n\n\n`);
+		})
+		.catch((e) => {
+			console.log(`------------------ERROR------------------`);
+			console.error(e);
+			console.log(`------------------ERROR------------------`);
+			debugger;
+		})
+		.finally(() => {
+			console.log('have a wonderful day :)');
+			openFinder(path.resolve("./data"));
+		});
+
+}
+
+else {
+	module.exports = main;
+}
+
+
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "scripts": {