make-mp-data 1.0.15 → 1.0.17

package/.vscode/settings.json

@@ -1,5 +1,8 @@
 {
 	"cSpell.words": [
 		"unparse"
-	]
+	],
+	"jest.runMode": "on-demand",
+	"jest.jestCommandLine": "npm run test --",
+	"js/ts.implicitProjectConfig.checkJs": false,
 }

package/cli.js

@@ -25,38 +25,45 @@ DOCS: https://github.com/ak--47/make-mp-data`)
 		.command('$0', 'model mixpanel data', () => { })
 		.option("token", {
 			demandOption: false,
+			alias: 't',
 			describe: 'project token; if supplied data will be sent to mixpanel',
 			type: 'string'
 		})
 		.option("seed", {
-			demandOption: false,
+			demandOption: false,			
+			alias: 's',
 			describe: 'randomness seed; used to create distinct_ids',
 			type: 'string'
 		})
 		.option("format", {
 			demandOption: false,
 			default: 'csv',
+			alias: 'f',
 			describe: 'csv or json',
 			type: 'string'
 		})
 		.option("numDays", {
 			demandOption: false,
+			alias: 'd',
 			describe: 'number of days in past to model',
 			type: 'number',
 		})
 		.option("numUsers", {
 			demandOption: false,
+			alias: 'u',
 			describe: 'number of users to model',
 			type: 'number',
 		})
 		.option("numEvents", {
 			demandOption: false,
+			alias: 'e',
 			describe: 'number of events to model',
 			type: 'number',
 		})
 		.option("region", {
 			demandOption: false,
 			default: 'US',
+			alias: 'r',
 			describe: 'either US or EU',
 			type: 'string'
 		})

package/default.js

@@ -1,6 +1,6 @@
 const Chance = require('chance');
 const chance = new Chance();
-const { weightedRange, makeProducts, date } = require('./utils.js');
+const { weightedRange, makeProducts, date, generateEmoji } = require('./utils.js');
 
 const config = {
 	token: "",
@@ -36,7 +36,8 @@ const config = {
 			"event": "page view",
 			"weight": 10,
 			"properties": {
-				path: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
+				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
+				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
 			}
 		},
 		{
@@ -66,7 +67,8 @@ const config = {
 		}
 	],
 	superProps: {
-		platform: ["web", "mobile", "kiosk"],
+		platform: ["web", "mobile", "web", "mobile", "web", "kiosk"],
+		emotions: generateEmoji(),
 
 	},
 	/*
@@ -76,17 +78,16 @@ const config = {
 	userProps: {
 		title: chance.profession.bind(chance),
 		luckyNumber: weightedRange(42, 420),
-		servicesUsed: [["foo"], ["foo", "bar"], ["foo", "bar", "baz"], ["foo", "bar", "baz", "qux"], ["baz", "qux"], ["qux"]],
+		vibe: generateEmoji(),		
 		spiritAnimal: chance.animal.bind(chance)
 	},
 
 	scdProps: {
-		plan: ["free", "free", "free", "basic", "basic", "premium", "enterprise"],
+		plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
 		MRR: weightedRange(0, 10000, 1000, .15),
 		NPS: weightedRange(0, 10, 150, 2),
 		marketingOptIn: [true, true, false],
 		dateOfRenewal: date(100, false),
-
 	},
 
 	/*

package/e2e.test.js

@@ -0,0 +1,50 @@
+/* cSpell:disable */
+// @ts-nocheck
+/* eslint-disable no-undef */
+/* eslint-disable no-debugger */
+/* eslint-disable no-unused-vars */
+const { generate } = require('./index.js');
+require('dotenv').config();
+const { execSync } = require("child_process");
+const u = require('ak-tools');
+
+const timeout = 60000;
+
+
+describe('e2e', () => {
+
+	test('works as module', async () => {
+		console.log('MODULE TEST');
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(900);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
+	}, timeout);
+
+	test('works as CLI', async () => {
+		console.log('CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(5);
+	}, timeout);
+
+
+
+});
+
+
+
+afterEach(() => {
+
+});
+
+afterAll(() => {
+	console.log('clearing...');
+	execSync(`npm run prune`);
+	console.log('...files cleared 👍');
+});

package/index.js

@@ -1,11 +1,13 @@
 #! /usr/bin/env node
 
+
 /*
 make fake mixpanel data easily!
 by AK 
 ak@mixpanel.com
 */
 
+const RUNTIME = process.env.RUNTIME || "unspecified";
 const mp = require("mixpanel-import");
 const path = require("path");
 const Chance = require("chance");
@@ -27,14 +29,18 @@ const {
 	openFinder,
 	applySkew,
 	boxMullerRandom,
+	getUniqueKeys
 } = require("./utils.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
+// @ts-ignore
 Array.prototype.pickOne = pick;
 const NOW = dayjs().unix();
 
+/** @typedef {import('./types.d.ts').Config} Config */
+
 const PEAK_DAYS = [
 	dayjs().subtract(2, "day").unix(),
 	dayjs().subtract(3, "day").unix(),
@@ -48,7 +54,10 @@ const PEAK_DAYS = [
 	dayjs().subtract(29, "day").unix(),
 ];
 
-//our main program
+/**
+ * generates fake mixpanel data
+ * @param  {Config} config
+ */
 async function main(config) {
 	let {
 		seed = "every time a rug is micturated upon in this fair city...",
@@ -70,29 +79,30 @@ async function main(config) {
 		region = "US",
 		writeToDisk = false,
 	} = config;
-	
+
 	//ensure we have a token or are writing to disk
 	if (require.main === module) {
 		if (!token) {
 			if (!writeToDisk) {
 				writeToDisk = true;
+				config.writeToDisk = true;
 			}
 		}
 	}
-	
+
 	const uuidChance = new Chance(seed);
 
-	//the function which generates $distinct_id + $created
+	//the function which generates $distinct_id + $created, skewing towards the present
 	function uuid() {
 		const distinct_id = uuidChance.guid();
 		let z = boxMullerRandom();
-		const skew = chance.normal({ mean: 10, dev: 3 });		
-		z = applySkew(z, skew); 
+		const skew = chance.normal({ mean: 10, dev: 3 });
+		z = applySkew(z, skew);
 
 		// Scale and shift the normally distributed value to fit the range of days
-		const maxZ = integer(2, 4); 
-		const scaledZ = (z / maxZ + 1) / 2; // Scale to a 0-1 range
-		const daysAgoBorn = Math.round(scaledZ * (numDays - 1)) + 1; // Scale to 1-numDays range
+		const maxZ = integer(2, 4);
+		const scaledZ = (z / maxZ + 1) / 2;
+		const daysAgoBorn = Math.round(scaledZ * (numDays - 1)) + 1;
 
 		return {
 			distinct_id,
@@ -223,7 +233,14 @@ async function main(config) {
 			for (const writeData of datasetsToWrite) {
 				if (format === "csv") {
 					console.log(`writing ${path}`);
-					const csv = Papa.unparse(writeData, {});
+					const columns = getUniqueKeys(writeData);
+					//papa parse needs nested JSON stringified
+					writeData.forEach((e) => {
+						for (const key in e) {
+							if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
+						}
+					})
+					const csv = Papa.unparse(writeData, { columns });
 					await touch(path, csv);
 					console.log(`\tdone\n`);
 				} else {
@@ -238,13 +255,13 @@ async function main(config) {
 	const creds = { token };
 	/** @type {import('mixpanel-import').Options} */
 	const importOpts = {
+		region,
 		fixData: true,
 		verbose: false,
 		forceStream: true,
 		strict: false,
 		dryRun: false,
 		abridged: false,
-		region,
 	};
 	//send to mixpanel
 	if (token) {
@@ -252,6 +269,9 @@ async function main(config) {
 			console.log(`importing events to mixpanel...`);
 			const imported = await mp(creds, eventData, {
 				recordType: "event",
+				fixData: true,
+				fixJson: true,
+				strict: false,
 				...importOpts,
 			});
 			console.log(`\tsent ${comma(imported.success)} events\n`);
@@ -331,10 +351,10 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
 	const event = {
 		event: chosenEvent.event,
 		distinct_id,
-		$source: "AK's fake data generator",
+		$source: "AKsTimeSoup",
 	};
 
-	if (isFirstEvent) event.time = earliestTime;
+	if (isFirstEvent) event.time = dayjs.unix(earliestTime).toISOString();
 	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
 
 	const props = { ...chosenEvent.properties, ...superProps };
@@ -392,7 +412,7 @@ function buildFileNames(config) {
 }
 
 /**
- * timestamp generator with a twist
+ * essentially, a timestamp generator with a twist
  * @param {number} earliestTime - The earliest timestamp in Unix format.
  * @param {number} latestTime - The latest timestamp in Unix format.
  * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
@@ -432,8 +452,8 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 
 	// usually, ensure the event time is within business hours
 	if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
-
-	return eventTime;
+	
+	return dayjs.unix(eventTime).toISOString();
 }
 
 

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "make-mp-data",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
+  "types": "types.d.ts",
   "scripts": {
     "start": "node index.js",
     "prune": "rm ./data/*",
 	"go": "sh ./go.sh",
-	"post": "npm publish"
+	"post": "npm publish",
+	"test": "jest"
   },
   "repository": {
     "type": "git",

package/types.d.ts

@@ -0,0 +1,53 @@
+// types.d.ts
+import { Chance } from "chance";
+
+export interface Config {
+  token?: string;
+  seed?: string;
+  numDays?: number;
+  numEvents?: number;
+  numUsers?: number;
+  format?: "csv" | "json";
+  region?: string;
+  events?: EventConfig[];
+  superProps?: Record<string, string[]>; // Flexible for any string keys
+  userProps?: Record<string, any>; // Could be more specific based on actual usage
+  scdProps?: {
+    plan?: string[];
+    MRR?: number;
+    NPS?: number;
+    marketingOptIn?: boolean[];
+    dateOfRenewal?: Date;
+  };
+  groupKeys?: [string, number][];
+  groupProps?: Record<string, GroupProperty>; // Adjust according to usage
+  lookupTables?: LookupTable[];
+  writeToDisk?: boolean;
+}
+
+interface EventConfig {
+  event?: string;
+  weight?: number;
+  properties?: {
+    [key: string]: any; // Consider refining based on actual properties used
+  };
+  isFirstEvent?: boolean;
+}
+
+interface GroupProperty {
+  [key?: string]: any;
+}
+
+interface LookupTable {
+  key?: string;
+  entries?: number;
+  attributes?: {
+    category?: string[];
+    demand?: string[];
+    supply?: string[];
+    manufacturer?: () => string;
+    price?: number;
+    rating?: number;
+    reviews?: number;
+  };
+}

package/utils.js

@@ -20,12 +20,28 @@ function pick() {
 function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
 	const now = dayjs.utc();
 	return function () {
-		const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
-		let then;
-		if (isPast) then = now.subtract(when, 'day');
-		if (!isPast) then = now.add(when, 'day');
-		if (format) return then.format(format);
-		if (!format) return then.toISOString();
+		try {
+			const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
+			let then;
+			if (isPast) {
+				then = now.subtract(when, 'day')
+					.subtract(integer(0, 23), 'hour')
+					.subtract(integer(0, 59), 'minute')
+					.subtract(integer(0, 59), 'second');
+			}
+			if (!isPast) {
+				then = now.add(when, 'day')
+					.add(integer(0, 23), 'hour')
+					.add(integer(0, 59), 'minute')
+					.add(integer(0, 59), 'second');
+			}
+			if (format) return then?.format(format);
+			if (!format) return then?.toISOString();
+		}
+		catch (e) {
+			if (format) return now?.format(format);
+			if (!format) return now?.toISOString();
+		}
 	};
 }
 
@@ -74,6 +90,10 @@ function exhaust(arr) {
 
 
 function integer(min, max) {
+	if (min === max) {
+		return min;
+	}
+
 	if (min > max) {
 		return chance.integer({
 			min: max,
@@ -81,10 +101,14 @@ function integer(min, max) {
 		});
 	}
 
-	return chance.integer({
-		min: min,
-		max: max
-	});
+	if (min < max) {
+		return chance.integer({
+			min: min,
+			max: max
+		});
+	}
+
+	return 0;
 }
 
 function makeProducts() {
@@ -168,7 +192,6 @@ function progress(thing, p) {
 	process.stdout.write(`${thing} processed ... ${comma(p)}`);
 }
 
-
 function person(bornDaysAgo = 30) {
 	//names and photos
 	const gender = chance.pickone(['male', 'female']);
@@ -213,7 +236,27 @@ function openFinder(path, callback) {
 	});
 }
 
+function getUniqueKeys(data) {
+	const keysSet = new Set();
+	data.forEach(item => {
+		Object.keys(item).forEach(key => keysSet.add(key));
+	});
+	return Array.from(keysSet);
+}
 
+//makes a random-sized array of emojis
+function generateEmoji(max = 10, array = false) {
+	return function () {
+		const emojis = ['😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😈', '👿', '👦', '👧', '👨', '👩', '👴', '👵', '👶', '🧒', '👮', '👷', '💂', '🕵', '👩‍⚕️', '👨‍⚕️', '👩‍🌾', '👨‍🌾', '👩‍🍳', '👨‍🍳', '👩‍🎓', '👨‍🎓', '👩‍🎤', '👨‍🎤', '👩‍🏫', '👨‍🏫', '👩‍🏭', '👨‍🏭', '👩‍💻', '👨‍💻', '👩‍💼', '👨‍💼', '👩‍🔧', '👨‍🔧', '👩‍🔬', '👨‍🔬', '👩‍🎨', '👨‍🎨', '👩‍🚒', '👨‍🚒', '👩‍✈️', '👨‍✈️', '👩‍🚀', '👨‍🚀', '👩‍⚖️', '👨‍⚖️', '🤶', '🎅', '👸', '🤴', '👰', '🤵', '👼', '🤰', '🙇', '💁', '🙅', '🙆', '🙋', '🤦', '🤷', '🙎', '🙍', '💇', '💆', '🕴', '💃', '🕺', '🚶', '🏃', '🤲', '👐', '🙌', '👏', '🤝', '👍', '👎', '👊', '✊', '🤛', '🤜', '🤞', '✌️', '🤟', '🤘', '👌', '👈', '👉', '👆', '👇', '☝️', '✋', '🤚', '🖐', '🖖', '👋', '🤙', '💪', '🖕', '✍️', '🤳', '💅', '👂', '👃', '👣', '👀', '👁', '🧠', '👅', '👄', '💋', '👓', '🕶', '👔', '👕', '👖', '🧣', '🧤', '🧥', '🧦', '👗', '👘', '👙', '👚', '👛', '👜', '👝', '🛍', '🎒', '👞', '👟', '👠', '👡', '👢', '👑', '👒', '🎩', '🎓', '🧢', '⛑', '📿', '💄', '💍', '💎', '🔇', '🔈', '🔉', '🔊', '📢', '📣', '📯', '🔔', '🔕', '🎼', '🎵', '🎶', '🎙', '🎚', '🎛', '🎤', '🎧', '📻', '🎷', '🎸', '🎹', '🎺', '🎻', '🥁', '📱', '📲', '💻', '🖥', '🖨', '🖱', '🖲', '🕹', '🗜', '💽', '💾', '💿', '📀', '📼', '📷', '📸', '📹', '🎥', '📽', '🎞', '📞', '☎️', '📟', '📠', '📺', '📻', '🎙', '📡', '🔍', '🔎', '🔬', '🔭', '📡', '💡', '🔦', '🏮', '📔', '📕', '📖', '📗', '📘', '📙', '📚', '📓', '📒', '📃', '📜', '📄', '📰', '🗞', '📑', '🔖', '🏷', '💰', '💴', '💵', '💶', '💷', '💸', '💳', '🧾', '💹', '💱', '💲', '✉️', '📧', '📨', '📩', '📤', '📥', '📦', '📫', '📪', '📬', '📭', '📮', '🗳', '✏️', '✒️', '🖋', '🖊', '🖌', '🖍', '📝', '💼', '📁', '📂', '🗂', '📅', '📆', '🗒', '🗓', '📇', '📈', '📉', '📊', '📋', '📌', '📍', '📎', '🖇', '📏', '📐', '✂️', '🗃', '🗄', '🗑', '🔒', '🔓', '🔏', '🔐', '🔑', '🗝', '🔨', '⛏', '⚒', '🛠', '🗡', '⚔️', '🔫', '🏹', '🛡', '🔧', '🔩', '⚙️', '🗜', '⚖️', '🔗', '⛓', '🧰', '🧲', '⚗️', '🧪', '🧫', '🧬', '🔬', '🔭', '📡', '💉', '💊', '🛏', '🛋', '🚪', '🚽', '🚿', '🛁', '🧴', '🧷', '🧹', '🧺', '🧻', '🧼', '🧽', '🧯', '🚬', '⚰️', '⚱️', '🗿', '🏺', '🧱', '🎈', '🎏', '🎀', '🎁', '🎊', '🎉', '🎎', '🏮', '🎐', '🧧', '✉️', '📩', '📨', '📧'];
+		let num = integer(1, max);
+		let arr = [];
+		for (let i = 0; i < num; i++) {
+			arr.push(chance.pickone(emojis));
+		}
+		if (array) return arr;
+		if (!array) return arr.join(', ');
+	};
+}
 
 module.exports = {
 	weightedRange,
@@ -230,4 +273,6 @@ module.exports = {
 	openFinder,
 	applySkew,
 	boxMullerRandom,
+	generateEmoji,
+	getUniqueKeys
 };