make-mp-data 1.4.5 → 1.5.1

package/package.json

@@ -1,25 +1,28 @@
 {
 	"name": "make-mp-data",
-	"version": "1.4.05",
+	"version": "1.5.01",
 	"description": "builds all mixpanel primitives for a given project",
 	"main": "index.js",
 	"types": "types.d.ts",
 	"scripts": {
-		"start": "node ./core/index.js",
-		"dev": "nodemon scratch.mjs --ignore /data",		
-		"prune": "rm -f ./data/* && rm -f ./tmp/*",
+		"start": "node ./index.js",
+		"dev": "nodemon scratch.mjs --ignore ./data/*",
+		"prune": "rm -f ./data/* && rm -f ./tmp/* && rm -f vscode-profile-*",
 		"post": "npm publish",
 		"test": "NODE_ENV=test jest --runInBand",
 		"coverage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --coverage && open ./tests/coverage/lcov-report/index.html",
-		"deps": "sh ./scripts/deps.sh",
-		"new": "sh ./scripts/new.sh"
+		"update:deps": "./scripts/update-deps.sh",
+		"new:dungeon": "./scripts/new-dungeon.sh",
+		"new:project": "node ./scripts/new-project.mjs",
+		"exp:benchmark": "node --no-warnings --experimental-vm-modules ./tests/benchmark/concurrency.mjs",
+		"exp:soup": "node ./tests/testSoup.mjs"
 	},
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/ak--47/make-mp-data.git"
 	},
 	"bin": {
-		"make-mp-data": "./core/index.js"
+		"make-mp-data": "./index.js"
 	},
 	"keywords": [
 		"mixpanel",
@@ -40,18 +43,21 @@
 	},
 	"homepage": "https://github.com/ak--47/make-mp-data#readme",
 	"dependencies": {
-		"ak-tools": "^1.0.58",
+		"ak-fetch": "^1.0.21",
+		"ak-tools": "^1.0.64",
 		"chance": "^1.1.11",
 		"chart.js": "^3.9.1",
 		"chartjs-node-canvas": "^4.1.6",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
-		"mixpanel-import": "^2.5.552",
+		"mixpanel-import": "^2.5.554",
+		"p-limit": "^3.1.0",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {
 		"@types/jest": "^29.5.12",
-		"jest": "^29.7.0"
+		"jest": "^29.7.0",
+		"nodemon": "^3.1.3"
 	},
 	"jest": {
 		"preset": "./tests/jest.config.js"

package/schemas/adspend.js

@@ -0,0 +1,96 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "foo bar",
+	numDays: 365, //how many days worth of data
+	numEvents: 10000000, //how many events
+	numUsers: 25000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	hasAdSpend: true,
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+		},
+		{
+			event: "bar",
+			weight: 9,
+		},
+		{
+			event: "baz",
+			weight: 8,
+		},
+		{
+			event: "qux",
+			weight: 7,
+		},
+		{
+			event: "garply",
+			weight: 6,
+		},
+		{
+			event: "durtle",
+			weight: 5,
+		},
+		{
+			event: "linny",
+			weight: 4,
+		},
+		{
+			event: "fonk",
+			weight: 3,
+		},
+		{
+			event: "crumn",
+			weight: 2,
+		},
+		{
+			event: "yak",
+			weight: 1,
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		number: integer,
+
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/anon.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer } = require('../core/utils');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
 
 
 
@@ -27,8 +27,8 @@ const config = {
 	numUsers: 10000, //how many users	
 	format: 'json', //csv or json
 	region: "US",
-	anonIds: true, //if true, anonymousIds are created for each user
-	sessionIds: false, //if true, sessionIds are created for each user
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	isAnonymous: true,
 	hasLocation: true, 
 	events: [

package/schemas/big.js

@@ -0,0 +1,160 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+/* cSpell:disable */
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "lets go",
+	numDays: 90, //how many days worth of data
+	numEvents: 100_000, //how many events
+	numUsers: 10_000, //how many users	
+	format: 'csv', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	hasLocation: true,
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		number: integer,
+
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {
+		"donk": ["dude", "man", "brok"],
+		"pronk": ["monk", "lonk", "aonk"],
+	},
+	mirrorProps: {},
+	groupKeys: [
+		["companies", 100_000],
+		["servers", 3_000_000],
+		["crews", 1000],
+
+	],
+	groupProps: {
+		companies: {
+			name: chance.company.bind(chance),
+			industry: ["tech", "finance", "healthcare", "education", "retail", "manufacturing", "entertainment", "government", "non-profit", "other"],
+		},
+		servers: {
+			name: chance.word.bind(chance),
+			ram: weighNumRange(4, 64, .25),
+			cpu: weighNumRange(1, 16, .25),
+
+		},
+		crews: {
+			name: chance.word.bind(chance),
+			department: ["engineering", "design", "marketing", "sales", "finance", "hr", "legal", "operations", "support", "other"],
+			size: weighNumRange(5, 50, .25)
+		}
+	},
+	lookupTables: [{
+			key: "product_id",
+			entries: 2_000_000,
+			attributes: {
+				category: [
+					"Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden & Tools", "Pet Supplies", "Food & Grocery", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art"
+				],
+				"demand": ["high", "medium", "medium", "low"],
+				"supply": ["high", "medium", "medium", "low"],
+				"manufacturer": chance.company.bind(chance),
+				"price": weighNumRange(5, 500, .25),
+				"rating": weighNumRange(1, 5),
+				"reviews": weighNumRange(0, 35)
+			}
+
+		},
+		{
+			key: "video_id",
+			entries: 10_000_000,
+			attributes: {
+				isFlagged: [true, false, false, false, false],
+				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
+				uploader_id: chance.guid.bind(chance),
+				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
+				thumbs: weighNumRange(0, 35),
+				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
+			}
+
+		}],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/business.js

@@ -0,0 +1,327 @@
+/**
+ * This is the default configuration file for the data generator in COMPLEX mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const { weighNumRange, date, integer, pickAWinner, exhaust } = require('../components/utils.js');
+const u = require('ak-tools');
+
+const channel_ids = [...Array(1234).keys()].map(i => i + 1).map(n => `channel_id_${n}`);
+const channel_names = chance.n(u.makeName, 1234);
+const video_ids = [...Array(50000).keys()].map(i => i + 1).map(n => n.toString());
+const video_names = chance.n(u.makeName, 50000);
+
+const EVENTS = 50_000
+const USERS = EVENTS / 100
+
+
+/** @type {import('../types.js').Config} */
+const config = {
+	token: "",	
+	seed: "it's business time...",
+	numDays: 90, //how many days worth of data
+	numEvents: EVENTS, //how many events
+	numUsers: USERS, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+
+	hasAvatar: false,
+	makeChart: true,
+
+	batchSize: 500_000,
+	concurrency: 500,
+
+	funnels: [
+		{
+			sequence: ["watch video", "like video", "subscribe", "purchase video"],
+			conversionRate: 35,
+			props: {
+				channel_id: pickAWinner(channel_ids),
+				video_id: weighNumRange(1, 50000, 1.4),
+				category: pickAWinner(["funny", "educational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"]),
+				isFeatured: () => { chance.bool({ likelihood: 25 }); },
+			}
+		},
+		{
+			sequence: ["watch video", "dislike video"],
+			conversionRate: 10,
+			order: "sequential",
+			props: {
+				channel_id: pickAWinner(channel_ids),
+				video_id: weighNumRange(1, 50000, .67),
+				category: pickAWinner(["funny", "educational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"]),
+				isFeatured: () => { chance.bool({ likelihood: 25 }); },
+			}
+		},
+	],
+	events: [
+		{
+			"event": "watch video",
+			"weight": 75,
+			"properties": {
+				"#hashtags": makeHashTags,
+				"watch time (sec)": weighNumRange(10, 600, .25),
+				"quality": pickAWinner(["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"], 1)
+			}
+		},
+		{
+			"event": "like video",
+			"weight": 30,
+			"properties": {
+
+			}
+		},
+		{
+			"event": "dislike video",
+			"weight": 15,
+			"properties": {
+
+			}
+		},
+		{
+			"event": "subscribe",
+			"weight": 10,
+			"properties": {
+				"UI": pickAWinner(["button", "link", "modal", "menu"]),
+			}
+		},
+		{
+			"event": "search",
+			"weight": 10,
+			"properties": {
+				term: () => { return chance.word(); },
+				"# results": weighNumRange(1, 100, .25),
+				"UI": pickAWinner(["button", "link", "modal", "menu"]),
+			}
+		},
+		{
+			"event": "comment",
+			"weight": 6,
+			"properties": {
+				length: weighNumRange(1, 500, .25),
+				video_id: weighNumRange(1, 50000, .72),
+				"has replies": [true, false, false, false, false],
+				"has photos": [true, false, false, false, false],
+
+			}
+		},
+		{
+			"event": "save video",
+			"weight": 17,
+			"properties": {
+				video_id: weighNumRange(1, 50000, 1.4),
+				UI: pickAWinner(["toolbar", "menu", "keyboard"])
+
+			}
+		},
+		{
+			"event": "create playlist",
+			"weight": 5,
+			"properties": {
+				"# of videos": weighNumRange(1, 100, .25),
+				"UI": pickAWinner(["toolbar", "menu", "keyboard"]),
+				"visibility": pickAWinner(["public", "private", "unlisted"]),
+			}
+		},
+		{
+			"event": "purchase video",
+			"weight": 12,
+			"properties": {
+				video_id: weighNumRange(1, 50000, 1.4),
+				basket: makeProducts(5),
+
+
+			}
+		},
+		{
+			"event": "support ticket",
+			"weight": 10,
+			"properties": {
+				description: chance.sentence.bind(chance),
+				severity: ["low", "medium", "high"],
+				ticket_id: chance.guid.bind(chance)
+			}
+		},
+		{
+			"event": "app error",
+			"weight": 15,
+			"properties": {
+				code: pickAWinner(["404", "500", "403", "401", "400", "503", "504", "429"]),
+				error: chance.sentence.bind(chance),
+				component: pickAWinner(["video player", "search", "comment", "profile", "settings", "billing", "support"]),
+			}
+		},
+		{
+			"event": "sign up",
+			"isFirstEvent": true,
+			"weight": 0,
+			"properties": {
+
+			}
+		}
+	],
+	superProps: {
+
+
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		experiment: designExperiment(),
+		spiritAnimal: ["unicorn", "dragon", "phoenix", "sasquatch", "yeti", "kraken", "jackalope", "thunderbird", "mothman", "nessie", "chupacabra", "jersey devil", "bigfoot", "weindgo", "bunyip", "mokele-mbembe", "tatzelwurm", "megalodon"],
+		
+		ip: chance.ip.bind(chance),
+
+	},
+
+	/** each generates it's own table */
+	scdProps: {
+		plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
+		MRR: weighNumRange(0, 10000, .15),
+		NPS: weighNumRange(0, 10, 2, 150),
+	},
+
+	mirrorProps: {
+
+	},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [
+		['channel_id', 1234, ["save video", "comment", "watch video", "purchase video", "like video", "dislike video", "subscribe"]],
+
+	],
+	groupProps: {
+		channel_id: {
+			"name": exhaust(channel_names),
+			"viewers": weighNumRange(5, 500, .25),
+			"rating": weighNumRange(1, 5),
+			"reviews": weighNumRange(0, 35)
+		}
+
+	},
+
+	lookupTables: [
+		{
+			key: "video_id",
+			entries: 50000,
+			attributes: {
+				isFlagged: [true, false, false, false, false],
+				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
+				uploader_id: chance.guid.bind(chance),
+				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
+				thumbs: weighNumRange(0, 4000, .25),
+				video_name: exhaust(video_names),
+				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
+			}
+
+		}
+	],
+
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+function makeHashTags() {
+	const possibleHashtags = [];
+	for (let i = 0; i < 20; i++) {
+		possibleHashtags.push('#' + u.makeName(2, ''));
+	}
+
+	const numHashtags = integer(integer(1, 5), integer(5, 10));
+	const hashtags = [];
+	for (let i = 0; i < numHashtags; i++) {
+		hashtags.push(chance.pickone(possibleHashtags));
+	}
+	return [hashtags];
+};
+
+function makeProducts(maxItems = 10) {
+	return function () {
+		const categories = ["Device Accessories", "eBooks", "Automotive", "Baby Products", "Beauty", "Books", "Camera & Photo", "Cell Phones & Accessories", "Collectible Coins", "Consumer Electronics", "Entertainment Collectibles", "Fine Art", "Grocery & Gourmet Food", "Health & Personal Care", "Home & Garden", "Independent Design", "Industrial & Scientific", "Accessories", "Major Appliances", "Music", "Musical Instruments", "Office Products", "Outdoors", "Personal Computers", "Pet Supplies", "Software", "Sports", "Sports Collectibles", "Tools & Home Improvement", "Toys & Games", "Video, DVD & Blu-ray", "Video Games", "Watches"];
+		const slugs = ['/sale/', '/featured/', '/home/', '/search/', '/wishlist/', '/'];
+		const assetExtension = ['.png', '.jpg', '.jpeg', '.heic', '.mp4', '.mov', '.avi'];
+		const data = [];
+		const numOfItems = integer(1, 12);
+
+		for (var i = 0; i < numOfItems; i++) {
+			const category = chance.pickone(categories);
+			const slug = chance.pickone(slugs);
+			const asset = chance.pickone(assetExtension);
+			const product_id = chance.guid();
+			const price = integer(1, 300);
+			const quantity = integer(1, 5);
+
+			const item = {
+				product_id: product_id,
+				sku: integer(11111, 99999),
+				amount: price,
+				quantity: quantity,
+				value: price * quantity,
+				featured: chance.pickone([true, false]),
+				category: category,
+				urlSlug: slug + category,
+				asset: `${category}-${integer(1, 20)}${asset}`
+			};
+
+			data.push(item);
+		}
+
+		return [data];
+	};
+};
+
+
+function designExperiment() {
+	return function () {
+		const variants = ["A", "B", "C", "Control"];
+		const variant = chance.pickone(variants);
+		const experiments = ["no password", "social sign in", "new tutorial", "new search"];
+		const experiment = chance.pickone(experiments);
+		const multi_variates = ["A/B", "A/B/C", "A/B/C/D", "Control"];
+		const multi_variate = chance.pickone(multi_variates);
+		const impression_id = chance.guid();
+
+
+
+		const chosen = {
+			variant,
+			experiment,
+			multi_variate,
+			impression_id
+		};
+
+		return [chosen];
+	};
+}
+
+
+
+
+
+
+module.exports = config;