make-mp-data 1.0.14 → 1.0.16

package/.vscode/settings.json

@@ -1,5 +1,8 @@
 {
 	"cSpell.words": [
 		"unparse"
-	]
+	],
+	"jest.runMode": "on-demand",
+	"jest.jestCommandLine": "npm run test --",
+	"js/ts.implicitProjectConfig.checkJs": false,
 }

package/cli.js

@@ -25,41 +25,61 @@ DOCS: https://github.com/ak--47/make-mp-data`)
 		.command('$0', 'model mixpanel data', () => { })
 		.option("token", {
 			demandOption: false,
+			alias: 't',
 			describe: 'project token; if supplied data will be sent to mixpanel',
 			type: 'string'
 		})
 		.option("seed", {
-			demandOption: false,
+			demandOption: false,			
+			alias: 's',
 			describe: 'randomness seed; used to create distinct_ids',
 			type: 'string'
 		})
 		.option("format", {
 			demandOption: false,
 			default: 'csv',
+			alias: 'f',
 			describe: 'csv or json',
 			type: 'string'
 		})
 		.option("numDays", {
 			demandOption: false,
+			alias: 'd',
 			describe: 'number of days in past to model',
 			type: 'number',
 		})
 		.option("numUsers", {
 			demandOption: false,
+			alias: 'u',
 			describe: 'number of users to model',
 			type: 'number',
 		})
 		.option("numEvents", {
 			demandOption: false,
+			alias: 'e',
 			describe: 'number of events to model',
 			type: 'number',
 		})
 		.option("region", {
 			demandOption: false,
 			default: 'US',
+			alias: 'r',
 			describe: 'either US or EU',
 			type: 'string'
 		})
+		.option("writeToDisk", {
+			demandOption: false,
+			default: true,
+			describe: 'write data to disk',
+			alias: 'w',
+			type: 'boolean',
+			coerce: (value) => {
+				if (typeof value === 'string') {
+					return value.toLowerCase() === 'true';
+				}
+				return value;
+			}
+		})
 		.help()
 		.wrap(null)
 		.argv;

package/default.js

@@ -1,6 +1,6 @@
 const Chance = require('chance');
 const chance = new Chance();
-const { weightedRange, makeProducts, date } = require('./utils.js');
+const { weightedRange, makeProducts, date, generateEmoji } = require('./utils.js');
 
 const config = {
 	token: "",
@@ -36,7 +36,8 @@ const config = {
 			"event": "page view",
 			"weight": 10,
 			"properties": {
-				path: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
+				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
+				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
 			}
 		},
 		{
@@ -66,7 +67,8 @@ const config = {
 		}
 	],
 	superProps: {
-		platform: ["web", "mobile", "kiosk"],
+		platform: ["web", "mobile", "web", "mobile", "web", "kiosk"],
+		emotions: generateEmoji(),
 
 	},
 	/*
@@ -76,17 +78,16 @@ const config = {
 	userProps: {
 		title: chance.profession.bind(chance),
 		luckyNumber: weightedRange(42, 420),
-		servicesUsed: [["foo"], ["foo", "bar"], ["foo", "bar", "baz"], ["foo", "bar", "baz", "qux"], ["baz", "qux"], ["qux"]],
+		vibe: generateEmoji(),		
 		spiritAnimal: chance.animal.bind(chance)
 	},
 
 	scdProps: {
-		plan: ["free", "free", "free", "basic", "basic", "premium", "enterprise"],
+		plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
 		MRR: weightedRange(0, 10000, 1000, .15),
 		NPS: weightedRange(0, 10, 150, 2),
 		marketingOptIn: [true, true, false],
 		dateOfRenewal: date(100, false),
-
 	},
 
 	/*

package/e2e.test.js

@@ -0,0 +1,50 @@
+/* cSpell:disable */
+// @ts-nocheck
+/* eslint-disable no-undef */
+/* eslint-disable no-debugger */
+/* eslint-disable no-unused-vars */
+const { generate } = require('./index.js');
+require('dotenv').config();
+const { execSync } = require("child_process");
+const u = require('ak-tools');
+
+const timeout = 60000;
+
+
+describe('e2e', () => {
+
+	test('works as module', async () => {
+		console.log('MODULE TEST');
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(900);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
+	}, timeout);
+
+	test('works as CLI', async () => {
+		console.log('CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(5);
+	}, timeout);
+
+
+
+});
+
+
+
+afterEach(() => {
+
+});
+
+afterAll(() => {
+	console.log('clearing...');
+	execSync(`npm run prune`);
+	console.log('...files cleared 👍');
+});

package/index.js

@@ -1,11 +1,13 @@
 #! /usr/bin/env node
 
+
 /*
 make fake mixpanel data easily!
 by AK 
 ak@mixpanel.com
 */
 
+const RUNTIME = process.env.RUNTIME || "unspecified";
 const mp = require("mixpanel-import");
 const path = require("path");
 const Chance = require("chance");
@@ -25,23 +27,37 @@ const {
 	range,
 	exhaust,
 	openFinder,
+	applySkew,
+	boxMullerRandom,
+	getUniqueKeys
 } = require("./utils.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
-const cliParams = require("./cli.js");
-
 dayjs.extend(utc);
+const cliParams = require("./cli.js");
+// @ts-ignore
 Array.prototype.pickOne = pick;
-const now = dayjs().unix();
-const dayInSec = 86400;
+const NOW = dayjs().unix();
+
+/** @typedef {import('./types.d.ts').Config} Config */
+
 const PEAK_DAYS = [
-	dayjs().subtract(1, "day").unix(),
+	dayjs().subtract(2, "day").unix(),
+	dayjs().subtract(3, "day").unix(),
 	dayjs().subtract(5, "day").unix(),
-	dayjs().subtract(10, "day").unix(),
-	dayjs().subtract(15, "day").unix(),
+	dayjs().subtract(7, "day").unix(),
+	dayjs().subtract(11, "day").unix(),
+	dayjs().subtract(13, "day").unix(),
+	dayjs().subtract(17, "day").unix(),
+	dayjs().subtract(19, "day").unix(),
+	dayjs().subtract(23, "day").unix(),
+	dayjs().subtract(29, "day").unix(),
 ];
 
-//our main program
+/**
+ * generates fake mixpanel data
+ * @param  {Config} config
+ */
 async function main(config) {
 	let {
 		seed = "every time a rug is micturated upon in this fair city...",
@@ -63,13 +79,31 @@ async function main(config) {
 		region = "US",
 		writeToDisk = false,
 	} = config;
-	if (require.main === module) writeToDisk = true;
+
+	//ensure we have a token or are writing to disk
+	if (require.main === module) {
+		if (!token) {
+			if (!writeToDisk) {
+				writeToDisk = true;
+				config.writeToDisk = true;
+			}
+		}
+	}
+
 	const uuidChance = new Chance(seed);
 
-	//the function which generates $distinct_id
+	//the function which generates $distinct_id + $created, skewing towards the present
 	function uuid() {
 		const distinct_id = uuidChance.guid();
-		const daysAgoBorn = chance.integer({ min: 1, max: numDays });
+		let z = boxMullerRandom();
+		const skew = chance.normal({ mean: 10, dev: 3 });
+		z = applySkew(z, skew);
+
+		// Scale and shift the normally distributed value to fit the range of days
+		const maxZ = integer(2, 4);
+		const scaledZ = (z / maxZ + 1) / 2;
+		const daysAgoBorn = Math.round(scaledZ * (numDays - 1)) + 1;
+
 		return {
 			distinct_id,
 			...person(daysAgoBorn),
@@ -199,7 +233,14 @@ async function main(config) {
 			for (const writeData of datasetsToWrite) {
 				if (format === "csv") {
 					console.log(`writing ${path}`);
-					const csv = Papa.unparse(writeData, {});
+					const columns = getUniqueKeys(writeData);
+					//papa parse needs nested JSON stringified
+					writeData.forEach((e) => {
+						for (const key in e) {
+							if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
+						}
+					})
+					const csv = Papa.unparse(writeData, { columns });
 					await touch(path, csv);
 					console.log(`\tdone\n`);
 				} else {
@@ -214,13 +255,13 @@ async function main(config) {
 	const creds = { token };
 	/** @type {import('mixpanel-import').Options} */
 	const importOpts = {
+		region,
 		fixData: true,
 		verbose: false,
 		forceStream: true,
 		strict: false,
 		dryRun: false,
 		abridged: false,
-		region,
 	};
 	//send to mixpanel
 	if (token) {
@@ -300,25 +341,18 @@ function makeSCD(props, distinct_id, mutations, $created) {
 	return scdEntries;
 }
 
-function makeEvent(
-	distinct_id,
-	earliestTime,
-	events,
-	superProps,
-	groupKeys,
-	isFirstEvent = false
-) {
+function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
 	let chosenEvent = events.pickOne();
 	if (typeof chosenEvent === "string")
 		chosenEvent = { event: chosenEvent, properties: {} };
 	const event = {
 		event: chosenEvent.event,
 		distinct_id,
-		$source: "AK's fake data generator",
+		$source: "AKsTimeSoup",
 	};
 
 	if (isFirstEvent) event.time = earliestTime;
-	if (!isFirstEvent) event.time = customTimeDistribution(earliestTime, now, PEAK_DAYS);
+	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
 
 	const props = { ...chosenEvent.properties, ...superProps };
 
@@ -375,16 +409,16 @@ function buildFileNames(config) {
 }
 
 /**
- * Generates a random timestamp with higher likelihood on peak days and typical business hours.
+ * essentially, a timestamp generator with a twist
  * @param {number} earliestTime - The earliest timestamp in Unix format.
  * @param {number} latestTime - The latest timestamp in Unix format.
  * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
  * @returns {number} - The generated event timestamp in Unix format.
  */
-function customTimeDistribution(earliestTime, latestTime, peakDays) {
+function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 	// Define business hours
-	const peakStartHour = 8; // 8 AM
-	const peakEndHour = 18; // 6 PM
+	const peakStartHour = 4; // 4 AM
+	const peakEndHour = 23; // 11 PM
 	const likelihoodOfPeakDay = chance.integer({ min: integer(5, 42), max: integer(43, 69) }); // Randomize likelihood with CHAOS!~~
 
 	// Select a day, with a preference for peak days
@@ -393,7 +427,7 @@ function customTimeDistribution(earliestTime, latestTime, peakDays) {
 		selectedDay = peakDays.length > 0 ? chance.pickone(peakDays) : integer(earliestTime, latestTime);
 	} else {
 		// Introduce minor peaks by allowing some events to still occur during business hours
-		selectedDay = chance.bool({ likelihood: 20 }) // 20% chance to simulate a minor peak on a non-peak day
+		selectedDay = chance.bool({ likelihood: integer(1, 42) })
 			? chance.pickone(peakDays)
 			: integer(earliestTime, latestTime);
 	}
@@ -407,12 +441,14 @@ function customTimeDistribution(earliestTime, latestTime, peakDays) {
 	let eventTime;
 	if (selectedDay === peakDays[0]) {
 		// Use a skewed distribution for peak days
-		eventTime = chance.normal({ mean: (businessEnd + businessStart) / 2, dev: (businessEnd - businessStart) / 8 });
+		eventTime = chance.normal({ mean: (businessEnd + businessStart) / integer(1, 4), dev: (businessEnd - businessStart) / integer(2, 8) });
 	} else {
 		// For non-peak days, use a uniform distribution to add noise
-		eventTime = integer(businessStart, businessEnd);
+		eventTime = integer(integer(businessStart, businessEnd), integer(businessStart, businessEnd));
 	}
-	eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd); // Ensure time is within business hours
+
+	// usually, ensure the event time is within business hours
+	if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
 
 	return eventTime;
 }
@@ -422,7 +458,7 @@ function customTimeDistribution(earliestTime, latestTime, peakDays) {
 // this is for CLI
 if (require.main === module) {
 	const args = cliParams();
-	const { token, seed, format, numDays, numUsers, numEvents, region } = args;
+	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk } = args;
 	const suppliedConfig = args._[0];
 
 	//if the user specifics an separate config file
@@ -443,6 +479,7 @@ if (require.main === module) {
 	if (numUsers) config.numUsers = numUsers;
 	if (numEvents) config.numEvents = numEvents;
 	if (region) config.region = region;
+	if (writeToDisk) config.writeToDisk = writeToDisk;
 
 	main(config)
 		.then((data) => {

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "make-mp-data",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
+  "types": "types.d.ts",
   "scripts": {
     "start": "node index.js",
     "prune": "rm ./data/*",
 	"go": "sh ./go.sh",
-	"post": "npm publish"
+	"post": "npm publish",
+	"test": "jest"
   },
   "repository": {
     "type": "git",

package/types.d.ts

@@ -0,0 +1,53 @@
+// types.d.ts
+import { Chance } from "chance";
+
+export interface Config {
+  token?: string;
+  seed?: string;
+  numDays?: number;
+  numEvents?: number;
+  numUsers?: number;
+  format?: "csv" | "json";
+  region?: string;
+  events?: EventConfig[];
+  superProps?: Record<string, string[]>; // Flexible for any string keys
+  userProps?: Record<string, any>; // Could be more specific based on actual usage
+  scdProps?: {
+    plan?: string[];
+    MRR?: number;
+    NPS?: number;
+    marketingOptIn?: boolean[];
+    dateOfRenewal?: Date;
+  };
+  groupKeys?: [string, number][];
+  groupProps?: Record<string, GroupProperty>; // Adjust according to usage
+  lookupTables?: LookupTable[];
+  writeToDisk?: boolean;
+}
+
+interface EventConfig {
+  event?: string;
+  weight?: number;
+  properties?: {
+    [key: string]: any; // Consider refining based on actual properties used
+  };
+  isFirstEvent?: boolean;
+}
+
+interface GroupProperty {
+  [key?: string]: any;
+}
+
+interface LookupTable {
+  key?: string;
+  entries?: number;
+  attributes?: {
+    category?: string[];
+    demand?: string[];
+    supply?: string[];
+    manufacturer?: () => string;
+    price?: number;
+    rating?: number;
+    reviews?: number;
+  };
+}

package/utils.js

@@ -20,12 +20,28 @@ function pick() {
 function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
 	const now = dayjs.utc();
 	return function () {
-		const when = chance.integer({ min: 0, max: inTheLast });
-		let then;
-		if (isPast) then = now.subtract(when, 'day');
-		if (!isPast) then = now.add(when, 'day');
-		if (format) return then.format(format);
-		if (!format) return then.toISOString();
+		try {
+			const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
+			let then;
+			if (isPast) {
+				then = now.subtract(when, 'day')
+					.subtract(integer(0, 23), 'hour')
+					.subtract(integer(0, 59), 'minute')
+					.subtract(integer(0, 59), 'second');
+			}
+			if (!isPast) {
+				then = now.add(when, 'day')
+					.add(integer(0, 23), 'hour')
+					.add(integer(0, 59), 'minute')
+					.add(integer(0, 59), 'second');
+			}
+			if (format) return then?.format(format);
+			if (!format) return then?.toISOString();
+		}
+		catch (e) {
+			if (format) return now?.format(format);
+			if (!format) return now?.toISOString();
+		}
 	};
 }
 
@@ -74,6 +90,10 @@ function exhaust(arr) {
 
 
 function integer(min, max) {
+	if (min === max) {
+		return min;
+	}
+
 	if (min > max) {
 		return chance.integer({
 			min: max,
@@ -81,10 +101,14 @@ function integer(min, max) {
 		});
 	}
 
-	return chance.integer({
-		min: min,
-		max: max
-	});
+	if (min < max) {
+		return chance.integer({
+			min: min,
+			max: max
+		});
+	}
+
+	return 0;
 }
 
 function makeProducts() {
@@ -168,7 +192,6 @@ function progress(thing, p) {
 	process.stdout.write(`${thing} processed ... ${comma(p)}`);
 }
 
-
 function person(bornDaysAgo = 30) {
 	//names and photos
 	const gender = chance.pickone(['male', 'female']);
@@ -213,7 +236,27 @@ function openFinder(path, callback) {
 	});
 }
 
+function getUniqueKeys(data) {
+	const keysSet = new Set();
+	data.forEach(item => {
+		Object.keys(item).forEach(key => keysSet.add(key));
+	});
+	return Array.from(keysSet);
+}
 
+//makes a random-sized array of emojis
+function generateEmoji(max = 10, array = false) {
+	return function () {
+		const emojis = ['😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😈', '👿', '👦', '👧', '👨', '👩', '👴', '👵', '👶', '🧒', '👮', '👷', '💂', '🕵', '👩‍⚕️', '👨‍⚕️', '👩‍🌾', '👨‍🌾', '👩‍🍳', '👨‍🍳', '👩‍🎓', '👨‍🎓', '👩‍🎤', '👨‍🎤', '👩‍🏫', '👨‍🏫', '👩‍🏭', '👨‍🏭', '👩‍💻', '👨‍💻', '👩‍💼', '👨‍💼', '👩‍🔧', '👨‍🔧', '👩‍🔬', '👨‍🔬', '👩‍🎨', '👨‍🎨', '👩‍🚒', '👨‍🚒', '👩‍✈️', '👨‍✈️', '👩‍🚀', '👨‍🚀', '👩‍⚖️', '👨‍⚖️', '🤶', '🎅', '👸', '🤴', '👰', '🤵', '👼', '🤰', '🙇', '💁', '🙅', '🙆', '🙋', '🤦', '🤷', '🙎', '🙍', '💇', '💆', '🕴', '💃', '🕺', '🚶', '🏃', '🤲', '👐', '🙌', '👏', '🤝', '👍', '👎', '👊', '✊', '🤛', '🤜', '🤞', '✌️', '🤟', '🤘', '👌', '👈', '👉', '👆', '👇', '☝️', '✋', '🤚', '🖐', '🖖', '👋', '🤙', '💪', '🖕', '✍️', '🤳', '💅', '👂', '👃', '👣', '👀', '👁', '🧠', '👅', '👄', '💋', '👓', '🕶', '👔', '👕', '👖', '🧣', '🧤', '🧥', '🧦', '👗', '👘', '👙', '👚', '👛', '👜', '👝', '🛍', '🎒', '👞', '👟', '👠', '👡', '👢', '👑', '👒', '🎩', '🎓', '🧢', '⛑', '📿', '💄', '💍', '💎', '🔇', '🔈', '🔉', '🔊', '📢', '📣', '📯', '🔔', '🔕', '🎼', '🎵', '🎶', '🎙', '🎚', '🎛', '🎤', '🎧', '📻', '🎷', '🎸', '🎹', '🎺', '🎻', '🥁', '📱', '📲', '💻', '🖥', '🖨', '🖱', '🖲', '🕹', '🗜', '💽', '💾', '💿', '📀', '📼', '📷', '📸', '📹', '🎥', '📽', '🎞', '📞', '☎️', '📟', '📠', '📺', '📻', '🎙', '📡', '🔍', '🔎', '🔬', '🔭', '📡', '💡', '🔦', '🏮', '📔', '📕', '📖', '📗', '📘', '📙', '📚', '📓', '📒', '📃', '📜', '📄', '📰', '🗞', '📑', '🔖', '🏷', '💰', '💴', '💵', '💶', '💷', '💸', '💳', '🧾', '💹', '💱', '💲', '✉️', '📧', '📨', '📩', '📤', '📥', '📦', '📫', '📪', '📬', '📭', '📮', '🗳', '✏️', '✒️', '🖋', '🖊', '🖌', '🖍', '📝', '💼', '📁', '📂', '🗂', '📅', '📆', '🗒', '🗓', '📇', '📈', '📉', '📊', '📋', '📌', '📍', '📎', '🖇', '📏', '📐', '✂️', '🗃', '🗄', '🗑', '🔒', '🔓', '🔏', '🔐', '🔑', '🗝', '🔨', '⛏', '⚒', '🛠', '🗡', '⚔️', '🔫', '🏹', '🛡', '🔧', '🔩', '⚙️', '🗜', '⚖️', '🔗', '⛓', '🧰', '🧲', '⚗️', '🧪', '🧫', '🧬', '🔬', '🔭', '📡', '💉', '💊', '🛏', '🛋', '🚪', '🚽', '🚿', '🛁', '🧴', '🧷', '🧹', '🧺', '🧻', '🧼', '🧽', '🧯', '🚬', '⚰️', '⚱️', '🗿', '🏺', '🧱', '🎈', '🎏', '🎀', '🎁', '🎊', '🎉', '🎎', '🏮', '🎐', '🧧', '✉️', '📩', '📨', '📧'];
+		let num = integer(1, max);
+		let arr = [];
+		for (let i = 0; i < num; i++) {
+			arr.push(chance.pickone(emojis));
+		}
+		if (array) return arr;
+		if (!array) return arr.join(', ');
+	};
+}
 
 module.exports = {
 	weightedRange,
@@ -227,5 +270,9 @@ module.exports = {
 	choose,
 	range,
 	exhaust,
-	openFinder
+	openFinder,
+	applySkew,
+	boxMullerRandom,
+	generateEmoji,
+	getUniqueKeys
 };