make-mp-data 1.3.4 → 1.4.0

package/package.json

@@ -1,53 +1,60 @@
 {
-  "name": "make-mp-data",
-  "version": "1.3.04",
-  "description": "builds all mixpanel primitives for a given project",
-  "main": "index.js",
-  "types": "types.d.ts",
-  "scripts": {
-    "start": "node index.js",
-    "dev": "./scripts/go.sh",
-    "complex": "nodemon index.js --complex --e 10000 --u 100",
-    "simple": "nodemon index.js  --simple --e 10000 --u 100",
-    "prune": "rm ./data/*",
-    "post": "npm publish",
-    "test": "NODE_ENV=test jest --runInBand",
-    "deps": "sh ./scripts/deps.sh"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/ak--47/make-mp-data.git"
-  },
-  "bin": {
-    "make-mp-data": "./index.js"
-  },
-  "keywords": [
-    "mixpanel",
-    "stream",
-    "analytics",
-    "tracking",
-    "server",
-    "CLI",
-    "datamart",
-    "scd 2",
-    "dummy data",
-    "fake data"
-  ],
-  "author": "ak@mixpanel.com",
-  "license": "ISC",
-  "bugs": {
-    "url": "https://github.com/ak--47/make-mp-data/issues"
-  },
-  "homepage": "https://github.com/ak--47/make-mp-data#readme",
-  "dependencies": {
-    "ak-tools": "^1.0.58",
-    "chance": "^1.1.11",
-    "dayjs": "^1.11.11",
-    "mixpanel-import": "^2.5.51",
-    "papaparse": "^5.4.1",
-    "yargs": "^17.7.2"
-  },
-  "devDependencies": {
-    "@types/jest": "^29.5.12"
-  }
-}
+	"name": "make-mp-data",
+	"version": "1.4.0",
+	"description": "builds all mixpanel primitives for a given project",
+	"main": "index.js",
+	"types": "types.d.ts",
+	"scripts": {
+		"start": "node index.js",
+		"dev": "nodemon scratch.mjs --ignore /data",
+		"complex": "nodemon index.js --complex --e 10000 --u 100",
+		"simple": "nodemon index.js  --simple --e 10000 --u 100",
+		"prune": "rm -f ./data/* && rm -f ./tmp/*",
+		"post": "npm publish",
+		"test": "NODE_ENV=test jest --runInBand",
+		"deps": "sh ./scripts/deps.sh",
+		"udacity": "nodemon index.js ./customers/udacity.js"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/ak--47/make-mp-data.git"
+	},
+	"bin": {
+		"make-mp-data": "./index.js"
+	},
+	"keywords": [
+		"mixpanel",
+		"stream",
+		"analytics",
+		"tracking",
+		"server",
+		"CLI",
+		"datamart",
+		"scd 2",
+		"dummy data",
+		"fake data"
+	],
+	"author": "ak@mixpanel.com",
+	"license": "ISC",
+	"bugs": {
+		"url": "https://github.com/ak--47/make-mp-data/issues"
+	},
+	"homepage": "https://github.com/ak--47/make-mp-data#readme",
+	"dependencies": {
+		"ak-tools": "^1.0.58",
+		"chance": "^1.1.11",
+		"chart.js": "^3.9.1",
+		"chartjs-node-canvas": "^4.1.6",
+		"dayjs": "^1.11.11",
+		"dotenv": "^16.4.5",
+		"mixpanel-import": "^2.5.54",
+		"yargs": "^17.7.2"
+	},
+	"devDependencies": {
+		"@types/jest": "^29.5.12",
+		"jest": "^29.7.0"
+	},
+	"jest": {
+		"preset": "./tests/jest.config.js"
+	}
+}

package/{models → schemas}/complex.js

@@ -28,7 +28,7 @@ const config = {
 			"event": "checkout",
 			"weight": 2,
 			"properties": {
-				amount: weightedRange(5, 500, 1000, .25),
+				amount: weightedRange(5, 500, .25, 1000),
 				currency: ["USD", "USD", "USD", "CAD", "EUR", "EUR", "BTC", "BTC", "ETH", "JPY"],
 				cart: makeProducts(12),
 			}
@@ -37,9 +37,9 @@ const config = {
 			"event": "add to cart",
 			"weight": 4,
 			"properties": {
-				amount: weightedRange(5, 500, 1000, .25),
+				amount: weightedRange(5, 500, .25, 1000),
 				qty: integer(1, 5),
-				product_id: weightedRange(1, 1000, 42, 1.4)
+				product_id: weightedRange(1, 1000, 1.4, 42)
 			}
 		},
 		{
@@ -56,10 +56,10 @@ const config = {
 			"properties": {
 				category: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
 				hashTags: makeHashTags,
-				watchTimeSec: weightedRange(10, 600, 1000, .25),
+				watchTimeSec: weightedRange(10, 600,.25,  1000),
 				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
 				format: ["mp4", "avi", "mov", "mpg"],
-				video_id: weightedRange(1, 50000, 420000, 1.4),
+				video_id: weightedRange(1, 50000, 1.4, 420000),
 
 			}
 		},
@@ -67,8 +67,8 @@ const config = {
 			"event": "comment",
 			"weight": 2,
 			"properties": {
-				length: weightedRange(1, 500, 1000, .25),
-				video_id: weightedRange(1, 50000, 420000, 1.4),
+				length: weightedRange(1, 500, .25, 1000),
+				video_id: weightedRange(1, 50000, 1.4,  420000),
 				has_replies: [true, false, false, false, false],
 				has_photo: [true, false, false, false, false],
 
@@ -78,7 +78,7 @@ const config = {
 			"event": "save video",
 			"weight": 4,
 			"properties": {
-				video_id: weightedRange(1, 50000, 420000, 1.4),
+				video_id: weightedRange(1, 50000, 1.4,  420000),
 				ui_control: ["toolbar", "menu", "keyboard"]
 
 
@@ -88,7 +88,7 @@ const config = {
 			"event": "view item",
 			"weight": 8,
 			"properties": {
-				product_id: weightedRange(1, 1000, 24, 3),
+				product_id: weightedRange(1,  24, 3, 1000),
 				colors: ["light", "dark", "custom", "dark"]
 			}
 		},
@@ -96,7 +96,7 @@ const config = {
 			"event": "save item",
 			"weight": 5,
 			"properties": {
-				product_id: weightedRange(1, 1000, 8, 12),
+				product_id: weightedRange(1, 1000, 12, 8 ),
 				colors: ["light", "dark", "custom", "dark"]
 			}
 		},
@@ -104,7 +104,7 @@ const config = {
 			"event": "support ticket",
 			"weight": 2,
 			"properties": {
-				product_id: weightedRange(1, 1000, 420, .6),
+				product_id: weightedRange(1, 1000, .6, 420),
 				description: chance.sentence.bind(chance),
 				severity: ["low", "medium", "high"],
 				ticket_id: chance.guid.bind(chance)
@@ -144,8 +144,8 @@ const config = {
 	/** each generates it's own table */
 	scdProps: {
 		plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
-		MRR: weightedRange(0, 10000, 1000, .15),
-		NPS: weightedRange(0, 10, 150, 2),
+		MRR: weightedRange(0, 10000,  .15, 1000),
+		NPS: weightedRange(0, 10,  2, 150),
 		subscribed: [true, true, true, true, true, true, false, false, false, false, "it's complicated"],
 		renewalDate: date(100, false),
 	},
@@ -155,7 +155,7 @@ const config = {
 		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
 		watchTimeSec: {
 			events: ["watch video"],
-			values: weightedRange(50, 1200, 247, 6)
+			values: weightedRange(50, 1200, 6 ,247)
 		}
 	},
 
@@ -170,16 +170,16 @@ const config = {
 	],
 	groupProps: {
 		company_id: {
-			$name: () => { return chance.company(); },
-			$email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
+			name: () => { return chance.company(); },
+			email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
 			"# of employees": weightedRange(3, 10000),
 			"industry": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
 			"segment": ["enterprise", "SMB", "mid-market"],
 			"products": [["core"], ["core"], ["core", "add-ons"], ["core", "pro-serve"], ["core", "add-ons", "pro-serve"], ["core", "BAA", "enterprise"], ["free"], ["free"], ["free", "addons"]],
 		},
 		room_id: {
-			$name: () => { return `#${chance.word({ length: integer(4, 24), capitalize: true })}`; },
-			$email: ["public", "private"],
+			name: () => { return `#${chance.word({ length: integer(4, 24), capitalize: true })}`; },
+			email: ["public", "private"],
 			"room provider": ["partner", "core", "core", "core"],
 			"room capacity": weightedRange(3, 1000000),
 			"isPublic": [true, false, false, false, false],

package/{models → schemas}/foobar.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { weighList, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 

package/schemas/funnels.js

@@ -0,0 +1,222 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weightedRange, date, integer } = require('../utils');
+
+const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
+
+const videoCategories = ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"];
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "simple is best",
+	numDays: 30, //how many days worth of data
+	numEvents: 50000, //how many events
+	numUsers: 500, //how many users	
+	format: 'csv', //csv or json
+	region: "US",
+	anonIds: false, //if true, anonymousIds are created for each user
+	sessionIds: false, //if true, sessionIds are created for each user
+
+	events: [
+		{
+			event: "checkout",
+			weight: 2,
+			properties: {
+				amount: weightedRange(5, 500, .25, 1000),
+				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
+				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
+				numItems: weightedRange(1, 10),
+
+			}
+		},
+		{
+			event: "add to cart",
+			weight: 4,
+			properties: {
+				amount: weightedRange(5, 500, .25, 1000),
+				rating: weightedRange(1, 5),
+				reviews: weightedRange(0, 35),
+				isFeaturedItem: [true, false, false],
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
+				dateItemListed: date(30, true, 'YYYY-MM-DD'),
+				itemId: integer(1000, 9999),
+			}
+		},
+		{
+			event: "page view",
+			weight: 10,
+			properties: {
+				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
+				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
+			}
+		},
+		{
+			event: "watch video",
+			weight: 8,
+			properties: {
+				videoCategory: pickAWinner(videoCategories, integer(0, 9)),
+				isFeaturedItem: [true, false, false],
+				watchTimeSec: weightedRange(10, 600, .25, 1000),
+				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
+				format: ["mp4", "avi", "mov", "mpg"],
+				uploader_id: chance.guid.bind(chance)
+
+			}
+		},
+		{
+			event: "view item",
+			weight: 8,
+			properties: {
+				isFeaturedItem: [true, false, false],
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
+				dateItemListed: date(30, true, 'YYYY-MM-DD'),
+				itemId: integer(1000, 9999),
+			}
+		},
+		{
+			event: "save item",
+			weight: 5,
+			properties: {
+				isFeaturedItem: [true, false, false],
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
+				dateItemListed: date(30, true, 'YYYY-MM-DD'),
+				itemId: integer(1000, 9999),
+			}
+		},
+		{
+			event: "sign up",
+			isFirstEvent: true,
+			weight: 0,
+			properties: {
+				CTA: ["sign up", "register", "join", "create account"]
+			}
+		}
+	],
+	funnels: [{
+		sequence: ["page view", "view item", "page view", "sign up"],
+		weight: 1,
+		isFirstFunnel: true,
+		order: "sequential",
+		conversionRate: 50,
+		timeToConvert: 2,
+		props: {
+			variants: ["A", "B", "C", "Control"],
+			flows: ["new", "existing", "loyal", "churned"],
+			flags: ["on", "off"],
+			experiment_ids: ["1234", "5678", "9012", "3456", "7890"],
+			multiVariate: [true, false]
+
+		},
+
+		// isInterruptedFunnel: false, // an interrupted funnel will have random events interspersed with the sequence
+		// fixedTimeFunnel: 30, // if set this funnel will occur for all users at the same time ['cart charged', 'charge complete']
+		// churn: {
+		// 	isChurnFunnel: true, //if the user completes this funnel, they churn
+		// 	probabilityToReturn: 0.1, //if the user churns, this is the probability they will return
+		// },
+	},
+	{
+		sequence: ["app install", "app open", "tutorial", "sign up"],
+		weight: 1,
+		isFirstFunnel: true,
+		order: "sequential",
+		conversionRate: 50,
+		timeToConvert: 2,
+		props: {
+			variants: ["A", "B", "C", "Control"],
+			flows: ["new", "existing", "loyal", "churned"],
+			flags: ["on", "off"],
+			experiment_ids: ["1234", "5678", "9012", "3456", "7890"],
+			multiVariate: [true, false]
+
+		}
+	},
+	{
+		sequence: ["page view", "view item", "add to cart", "add to cart", "checkout"],
+		weight: 3,
+		isFirstFunnel: false,
+		order: "sequential",
+		conversionRate: 70,
+		timeToConvert: 7 * 24,
+		props: {
+			variants: ["A", "B", "C", "Control"],
+			flows: ["new", "existing", "loyal", "churned"],
+			flags: ["on", "off"],
+			experiment_ids: ["1234", "5678", "9012", "3456", "7890"],
+			multiVariate: [true, false]
+
+		}
+	},
+	{
+		timeToConvert: 2,
+		conversionRate: 66,
+		sequence: ["foo", "bar", "baz", "qux"],
+	}, {
+		weight: 4,
+		sequence: ["video", "video", "attack", "defend", "click"],
+	}
+	],
+	superProps: {
+		platform: ["web", "mobile", "web", "mobile", "web", "web", "kiosk", "smartTV"],
+		currentTheme: ["light", "dark", "custom", "light", "dark"],
+		// emotions: generateEmoji(),
+
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weightedRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {
+		nps: [1, 1, 1, 4, 4, 4, 5, 5, 6, 7, 8, 9],
+		mrr: () => { weightedRange(10, 1000,.25,  1000); },
+	},
+	mirrorProps: {
+		isBot: { events: "*", values: [false, false, false, false, true] },
+		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
+		watchTimeSec: {
+			events: ["watch video"],
+			values: weightedRange(50, 1200, 6, 247)
+		}
+
+	},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [
+		["company_id", 5000],
+		["team_id", 500],
+		["department_id", 50]
+	],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/{models → schemas}/simple.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { weighList, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -24,7 +24,7 @@ const videoCategories = ["funny", "educational", "inspirational", "music", "news
 const config = {
 	token: "",
 	seed: "simple is best",
-	numDays: 30, //how many days worth of data
+	numDays: 30, //how many days worth1 of data
 	numEvents: 50000, //how many events
 	numUsers: 500, //how many users	
 	format: 'csv', //csv or json
@@ -37,7 +37,7 @@ const config = {
 			event: "checkout",
 			weight: 2,
 			properties: {
-				amount: weightedRange(5, 500, 1000, .25),
+				amount: weightedRange(5, 500, .25, 1000),
 				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
 				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
 				numItems: weightedRange(1, 10),
@@ -48,11 +48,11 @@ const config = {
 			event: "add to cart",
 			weight: 4,
 			properties: {
-				amount: weightedRange(5, 500, 1000, .25),
+				amount: weightedRange(5, 500, .25, 1000),
 				rating: weightedRange(1, 5),
 				reviews: weightedRange(0, 35),
 				isFeaturedItem: [true, false, false],
-				itemCategory: weighList(itemCategories, integer(0, 27)),
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
 				itemId: integer(1000, 9999),
 			}
@@ -69,9 +69,9 @@ const config = {
 			event: "watch video",
 			weight: 8,
 			properties: {
-				videoCategory: weighList(videoCategories, integer(0, 9)),
+				videoCategory: pickAWinner(videoCategories, integer(0, 9)),
 				isFeaturedItem: [true, false, false],
-				watchTimeSec: weightedRange(10, 600, 1000, .25),
+				watchTimeSec: weightedRange(10, 600, .25, 1000),
 				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
 				format: ["mp4", "avi", "mov", "mpg"],
 				uploader_id: chance.guid.bind(chance)
@@ -83,7 +83,7 @@ const config = {
 			weight: 8,
 			properties: {
 				isFeaturedItem: [true, false, false],
-				itemCategory: weighList(itemCategories, integer(0, 27)),
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
 				itemId: integer(1000, 9999),
 			}
@@ -93,7 +93,7 @@ const config = {
 			weight: 5,
 			properties: {
 				isFeaturedItem: [true, false, false],
-				itemCategory: weighList(itemCategories, integer(0, 27)),
+				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
 				itemId: integer(1000, 9999),
 			}
@@ -133,7 +133,7 @@ const config = {
 		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
 		watchTimeSec: {
 			events: ["watch video"],
-			values: weightedRange(50, 1200, 247, 6)
+			values: weightedRange(50, 1200, 6, 247)
 		}
 
 	},

package/scratch.mjs

@@ -0,0 +1,20 @@
+import main from "./index.js";
+import amir from './customers/amir.js';
+import simple from './schemas/simple.js';
+import execSync from 'child_process';
+
+
+/** @type {main.Config} */
+const spec = {	
+	...amir,
+	numUsers: 1000,
+	numEvents: 100000,
+	writeToDisk: false,
+	verbose: true,
+	makeChart: true,	
+};
+
+
+execSync.execSync('npm run prune');
+const { eventData, groupProfilesData, lookupTableData, mirrorEventData, scdTableData, userProfilesData, importResults } = await main(spec);
+debugger;