make-mp-data 1.0.13 → 1.0.15

package/cli.js

@@ -60,6 +60,19 @@ DOCS: https://github.com/ak--47/make-mp-data`)
 			describe: 'either US or EU',
 			type: 'string'
 		})
+		.option("writeToDisk", {
+			demandOption: false,
+			default: true,
+			describe: 'write data to disk',
+			alias: 'w',
+			type: 'boolean',
+			coerce: (value) => {
+				if (typeof value === 'string') {
+					return value.toLowerCase() === 'true';
+				}
+				return value;
+			}
+		})
 		.help()
 		.wrap(null)
 		.argv;

package/default.js

@@ -6,8 +6,8 @@ const config = {
 	token: "",
 	seed: "foo bar baz",
 	numDays: 30, //how many days worth of data
-	numEvents: 250000, //how many events
-	numUsers: 1500, //how many users	
+	numEvents: 100000, //how many events
+	numUsers: 1000, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
 

package/index.js

@@ -1,6 +1,5 @@
 #! /usr/bin/env node
 
-
 /*
 make fake mixpanel data easily!
 by AK 
@@ -25,21 +24,32 @@ const {
 	choose,
 	range,
 	exhaust,
-	openFinder
+	openFinder,
+	applySkew,
+	boxMullerRandom,
 } = require("./utils.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
-const cliParams = require('./cli.js');
-
-
 dayjs.extend(utc);
+const cliParams = require("./cli.js");
 Array.prototype.pickOne = pick;
-const now = dayjs().unix();
-const dayInSec = 86400;
+const NOW = dayjs().unix();
+
+const PEAK_DAYS = [
+	dayjs().subtract(2, "day").unix(),
+	dayjs().subtract(3, "day").unix(),
+	dayjs().subtract(5, "day").unix(),
+	dayjs().subtract(7, "day").unix(),
+	dayjs().subtract(11, "day").unix(),
+	dayjs().subtract(13, "day").unix(),
+	dayjs().subtract(17, "day").unix(),
+	dayjs().subtract(19, "day").unix(),
+	dayjs().subtract(23, "day").unix(),
+	dayjs().subtract(29, "day").unix(),
+];
 
 //our main program
 async function main(config) {
-
 	let {
 		seed = "every time a rug is micturated upon in this fair city...",
 		numEvents = 100000,
@@ -58,15 +68,32 @@ async function main(config) {
 		format = "csv",
 		token = null,
 		region = "US",
-		writeToDisk = false
+		writeToDisk = false,
 	} = config;
-	if (require.main === module) writeToDisk = true;
+	
+	//ensure we have a token or are writing to disk
+	if (require.main === module) {
+		if (!token) {
+			if (!writeToDisk) {
+				writeToDisk = true;
+			}
+		}
+	}
+	
 	const uuidChance = new Chance(seed);
 
-	//the function which generates $distinct_id
+	//the function which generates $distinct_id + $created
 	function uuid() {
 		const distinct_id = uuidChance.guid();
-		const daysAgoBorn = chance.integer({ min: 1, max: numDays });
+		let z = boxMullerRandom();
+		const skew = chance.normal({ mean: 10, dev: 3 });		
+		z = applySkew(z, skew); 
+
+		// Scale and shift the normally distributed value to fit the range of days
+		const maxZ = integer(2, 4); 
+		const scaledZ = (z / maxZ + 1) / 2; // Scale to a 0-1 range
+		const daysAgoBorn = Math.round(scaledZ * (numDays - 1)) + 1; // Scale to 1-numDays range
+
 		return {
 			distinct_id,
 			...person(daysAgoBorn),
@@ -84,7 +111,6 @@ async function main(config) {
 		}, [])
 		.filter((e) => !e.isFirstEvent);
 
-
 	const firstEvents = events.filter((e) => e.isFirstEvent);
 	const eventData = [];
 	const userProfilesData = [];
@@ -146,7 +172,7 @@ async function main(config) {
 			const group = {
 				[groupKey]: i,
 				...makeProfile(groupProps[groupKey]),
-				$distinct_id: i
+				$distinct_id: i,
 			};
 			groupProfiles.push(group);
 		}
@@ -168,7 +194,8 @@ async function main(config) {
 		}
 		lookupTableData.push({ key, data });
 	}
-	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder } = buildFileNames(config);
+	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder } =
+		buildFileNames(config);
 	const pairs = [
 		[eventFiles, eventData],
 		[userFiles, userProfilesData],
@@ -178,13 +205,20 @@ async function main(config) {
 	];
 	console.log("\n");
 
-	if (!writeToDisk && !token) return { eventData, userProfilesData, scdTableData, groupProfilesData, lookupTableData };
+	if (!writeToDisk && !token)
+		return {
+			eventData,
+			userProfilesData,
+			scdTableData,
+			groupProfilesData,
+			lookupTableData,
+		};
 	//write the files
 	for (const pair of pairs) {
 		const [paths, data] = pair;
 		for (const path of paths) {
 			let datasetsToWrite;
-			if (data?.[0]?.["key"]) datasetsToWrite = data.map(d => d.data);
+			if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
 			else datasetsToWrite = [data];
 			for (const writeData of datasetsToWrite) {
 				if (format === "csv") {
@@ -210,37 +244,47 @@ async function main(config) {
 		strict: false,
 		dryRun: false,
 		abridged: false,
-		region
-
+		region,
 	};
 	//send to mixpanel
 	if (token) {
 		if (eventData) {
 			console.log(`importing events to mixpanel...`);
-			const imported = await mp(creds, eventData, { recordType: "event", ...importOpts });
+			const imported = await mp(creds, eventData, {
+				recordType: "event",
+				...importOpts,
+			});
 			console.log(`\tsent ${comma(imported.success)} events\n`);
 			importResults.events = imported;
 		}
 		if (userProfilesData) {
 			console.log(`importing user profiles to mixpanel...`);
-			const imported = await mp(creds, userProfilesData, { recordType: "user", ...importOpts });
+			const imported = await mp(creds, userProfilesData, {
+				recordType: "user",
+				...importOpts,
+			});
 			console.log(`\tsent ${comma(imported.success)} user profiles\n`);
 			importResults.users = imported;
 		}
 		if (groupProfilesData) {
-
 			for (const groupProfiles of groupProfilesData) {
 				const groupKey = groupProfiles.key;
 				const data = groupProfiles.data;
 				console.log(`importing ${groupKey} profiles to mixpanel...`);
-				const imported = await mp({ token, groupKey }, data, { recordType: "group", ...importOpts });
+				const imported = await mp({ token, groupKey }, data, {
+					recordType: "group",
+					...importOpts,
+				});
 				console.log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
 				importResults.groups.push(imported);
 			}
 		}
 		console.log(`\n\n`);
 	}
-	return { import: importResults, files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder] };
+	return {
+		import: importResults,
+		files: [eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, folder],
+	};
 }
 
 function makeProfile(props, defaults) {
@@ -291,7 +335,7 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
 	};
 
 	if (isFirstEvent) event.time = earliestTime;
-	if (!isFirstEvent) event.time = integer(earliestTime, now);
+	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
 
 	const props = { ...chosenEvent.properties, ...superProps };
 
@@ -318,8 +362,8 @@ function buildFileNames(config) {
 	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
 	const extension = format === "csv" ? "csv" : "json";
 	const current = dayjs.utc().format("MM-DD-HH");
-	let writeDir = './';
-	if (config.writeToDisk)	writeDir = mkdir('./data');
+	let writeDir = "./";
+	if (config.writeToDisk) writeDir = mkdir("./data");
 
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `events-${current}.${extension}`)],
@@ -347,19 +391,57 @@ function buildFileNames(config) {
 	return writePaths;
 }
 
+/**
+ * timestamp generator with a twist
+ * @param {number} earliestTime - The earliest timestamp in Unix format.
+ * @param {number} latestTime - The latest timestamp in Unix format.
+ * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
+ * @returns {number} - The generated event timestamp in Unix format.
+ */
+function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
+	// Define business hours
+	const peakStartHour = 4; // 4 AM
+	const peakEndHour = 23; // 11 PM
+	const likelihoodOfPeakDay = chance.integer({ min: integer(5, 42), max: integer(43, 69) }); // Randomize likelihood with CHAOS!~~
+
+	// Select a day, with a preference for peak days
+	let selectedDay;
+	if (chance.bool({ likelihood: likelihoodOfPeakDay })) { // Randomized likelihood to pick a peak day
+		selectedDay = peakDays.length > 0 ? chance.pickone(peakDays) : integer(earliestTime, latestTime);
+	} else {
+		// Introduce minor peaks by allowing some events to still occur during business hours
+		selectedDay = chance.bool({ likelihood: integer(1, 42) })
+			? chance.pickone(peakDays)
+			: integer(earliestTime, latestTime);
+	}
+
+	// Normalize selectedDay to the start of the day
+	selectedDay = dayjs.unix(selectedDay).startOf('day').unix();
+
+	// Generate a random time within business hours with a higher concentration in the middle of the period
+	const businessStart = dayjs.unix(selectedDay).hour(peakStartHour).minute(0).second(0).unix();
+	const businessEnd = dayjs.unix(selectedDay).hour(peakEndHour).minute(0).second(0).unix();
+	let eventTime;
+	if (selectedDay === peakDays[0]) {
+		// Use a skewed distribution for peak days
+		eventTime = chance.normal({ mean: (businessEnd + businessStart) / integer(1, 4), dev: (businessEnd - businessStart) / integer(2, 8) });
+	} else {
+		// For non-peak days, use a uniform distribution to add noise
+		eventTime = integer(integer(businessStart, businessEnd), integer(businessStart, businessEnd));
+	}
+
+	// usually, ensure the event time is within business hours
+	if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
+
+	return eventTime;
+}
+
+
+
 // this is for CLI
 if (require.main === module) {
-
 	const args = cliParams();
-	const {
-		token,
-		seed,
-		format,
-		numDays,
-		numUsers,
-		numEvents,
-		region
-	} = args;
+	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk } = args;
 	const suppliedConfig = args._[0];
 
 	//if the user specifics an separate config file
@@ -380,7 +462,7 @@ if (require.main === module) {
 	if (numUsers) config.numUsers = numUsers;
 	if (numEvents) config.numEvents = numEvents;
 	if (region) config.region = region;
-
+	if (writeToDisk) config.writeToDisk = writeToDisk;
 
 	main(config)
 		.then((data) => {
@@ -388,18 +470,22 @@ if (require.main === module) {
 			const { events, groups, users } = data.import;
 			const files = data.files;
 			const folder = files.pop();
-			const groupBytes = groups.reduce((acc, group) => { return acc + group.bytes; }, 0);
-			const groupSuccess = groups.reduce((acc, group) => { return acc + group.success; }, 0);
+			const groupBytes = groups.reduce((acc, group) => {
+				return acc + group.bytes;
+			}, 0);
+			const groupSuccess = groups.reduce((acc, group) => {
+				return acc + group.success;
+			}, 0);
 			const bytes = events.bytes + groupBytes + users.bytes;
 			const stats = {
 				events: comma(events.success || 0),
 				users: comma(users.success || 0),
 				groups: comma(groupSuccess || 0),
-				bytes: bytesHuman(bytes || 0)
+				bytes: bytesHuman(bytes || 0),
 			};
 			if (bytes > 0) console.table(stats);
 			console.log(`\nfiles written to ${folder}...`);
-			console.log("\t" + files.flat().join('\n\t'));
+			console.log("\t" + files.flat().join("\n\t"));
 			console.log(`\n------------------SUMMARY------------------\n\n\n`);
 		})
 		.catch((e) => {
@@ -409,13 +495,10 @@ if (require.main === module) {
 			debugger;
 		})
 		.finally(() => {
-			console.log('have a wonderful day :)');
+			console.log("have a wonderful day :)");
 			openFinder(path.resolve("./data"));
 		});
-
-}
-
-else {
+} else {
 	module.exports = {
 		generate: main,
 		weightedRange,
@@ -429,9 +512,6 @@ else {
 		choose,
 		range,
 		exhaust,
-		openFinder
+		openFinder,
 	};
 }
-
-
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "scripts": {

package/utils.js

@@ -20,7 +20,7 @@ function pick() {
 function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
 	const now = dayjs.utc();
 	return function () {
-		const when = chance.integer({ min: 0, max: inTheLast });
+		const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
 		let then;
 		if (isPast) then = now.subtract(when, 'day');
 		if (!isPast) then = now.add(when, 'day');
@@ -227,5 +227,7 @@ module.exports = {
 	choose,
 	range,
 	exhaust,
-	openFinder
+	openFinder,
+	applySkew,
+	boxMullerRandom,
 };