make-mp-data 1.5.53 → 1.5.55

package/dungeons/big.js

@@ -1,160 +1,224 @@
-/**
- * This is the default configuration file for the data generator in SIMPLE mode
- * notice how the config object is structured, and see it's type definition in ./types.d.ts
- * feel free to modify this file to customize the data you generate
- * see helper functions in utils.js for more ways to generate data
- */
-
-
-/* cSpell:disable */
-
-
-
+const seed = "lets go big";
 const Chance = require('chance');
 const chance = new Chance();
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
-const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
+const { uid, comma, makeName } = require('ak-tools');
+const { pickAWinner, weighNumRange, integer, date, choose } = require('../components/utils');
+
+
+const eventsPerQuarter = 5_000_000_000 // ~5 billion
+const numQuarters = 8; // 24 months
+const parallelism = 5000; 
+const totalEvents = Math.floor((eventsPerQuarter * numQuarters) / parallelism);
+const eventPerUser = 500;
+const totalUsers = Math.floor(totalEvents / eventPerUser);
+const totalDays = (numQuarters * 90) + 10;
 
 /** @type {import('../types').Dungeon} */
 const config = {
 	token: "",
-	seed: "lets go",
-	numDays: 90, //how many days worth of data
-	numEvents: 100_000, //how many events
-	numUsers: 10_000, //how many users	
-	format: 'csv', //csv or json
+	seed: seed,
+	numDays: totalDays,
+	numEvents: totalEvents,
+	numUsers: totalUsers,
+	format: 'json', //csv or json
 	region: "US",
-	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasAnonIds: false, //if true, anonymousIds are created for each user
 	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	hasLocation: true,
-	events: [
+	hasAndroidDevices: false,
+	alsoInferFunnels: false,
+	batchSize: 2_000_000,
+	hasAvatar: false,
+	hasAdSpend: false,
+	hasBrowser: false,
+	hasCampaigns: false,
+	hasDesktopDevices: false,
+	hasIOSDevices: false,
+	writeToDisk: "gs://dungeon_master_4/big_data",
+	funnels: [
 		{
-			event: "foo",
-			weight: 10,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny", "fonk", "crumn", "yak"],
+			weight: 1,
+			order: "sequential"
 		},
 		{
-			event: "bar",
-			weight: 9,
-			properties: {}
+			"sequence": ["foo", "bar"],
+			weight: 25,
+			order: "sequential"
 		},
 		{
-			event: "baz",
-			weight: 8,
-			properties: {}
+			"sequence": ["foo", "bar", "baz"],
+			weight: 20,
+			order: "sequential"
 		},
 		{
-			event: "qux",
-			weight: 7,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux"],
+			weight: 15,
+			order: "sequential"
 		},
 		{
-			event: "garply",
-			weight: 6,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux", "garply"],
+			weight: 10,
+			order: "sequential"
 		},
 		{
-			event: "durtle",
-			weight: 5,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle"],
+			weight: 8,
+			order: "sequential"
 		},
 		{
-			event: "linny",
-			weight: 4,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny"],
+			weight: 6,
+			order: "sequential"
 		},
 		{
-			event: "fonk",
-			weight: 3,
-			properties: {}
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny", "fonk"],
+			weight: 4,
+			order: "sequential"
 		},
 		{
-			event: "crumn",
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny", "fonk", "crumn"],
 			weight: 2,
-			properties: {}
+			order: "sequential"
 		},
 		{
-			event: "yak",
+			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny", "fonk", "crumn", "yak"],
 			weight: 1,
-			properties: {}
+			order: "sequential"
 		}
-	],
-	superProps: {
-		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
-		number: integer,
 
+	],
+	events: [
+		{ event: "foo" },
+		{ event: "bar" },
+		{ event: "baz" },
+		{ event: "qux" },
+		{ event: "garply" },
+		{ event: "durtle" },
+		{ event: "linny" },
+		{ event: "fonk" },
+		{ event: "crumn" },
+		{ event: "yak" }
+	],
+	//? https://docs.mixpanel.com/docs/data-structure/property-reference/data-type
+	superProps: {		
+		"string": pickAWinner(["red", "orange", "yellow", "green", "blue", "indigo", "violet"]),
+		"number": integer,
+		"boolean": [true, true, false],
+		"date": () => date(90),
+		"string []": buildStringArray,
+		"number []": buildNumberArray,
+		"object {}": buildObjectProp,
+		"object [{}]": buildObjArrayProp
 	},
 	userProps: {
 		title: chance.profession.bind(chance),
 		luckyNumber: weighNumRange(42, 420),
-		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+		spiritAnimal: pickAWinner(["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]),
+		isHappyCustomer: pickAWinner([true, true, false]),
 	},
+	hook: function (record, type, meta) {
 
-	scdProps: {
-		"donk": ["dude", "man", "brok"],
-		"pronk": ["monk", "lonk", "aonk"],
-	},
-	mirrorProps: {},
-	groupKeys: [
-		["companies", 100_000],
-		["servers", 3_000_000],
-		["crews", 1000],
 
-	],
-	groupProps: {
-		companies: {
-			name: chance.company.bind(chance),
-			industry: ["tech", "finance", "healthcare", "education", "retail", "manufacturing", "entertainment", "government", "non-profit", "other"],
-		},
-		servers: {
-			name: chance.word.bind(chance),
-			ram: weighNumRange(4, 64, .25),
-			cpu: weighNumRange(1, 16, .25),
+		// if (type === "event") {
+		// 	debugger;
+		// }
 
-		},
-		crews: {
-			name: chance.word.bind(chance),
-			department: ["engineering", "design", "marketing", "sales", "finance", "hr", "legal", "operations", "support", "other"],
-			size: weighNumRange(5, 50, .25)
-		}
-	},
-	lookupTables: [{
-			key: "product_id",
-			entries: 2_000_000,
-			attributes: {
-				category: [
-					"Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden & Tools", "Pet Supplies", "Food & Grocery", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art"
-				],
-				"demand": ["high", "medium", "medium", "low"],
-				"supply": ["high", "medium", "medium", "low"],
-				"manufacturer": chance.company.bind(chance),
-				"price": weighNumRange(5, 500, .25),
-				"rating": weighNumRange(1, 5),
-				"reviews": weighNumRange(0, 35)
-			}
+		// if (type === "user") {
+
+		// }
+
+		// if (type === "funnel-post") {
+
+		// }
+
+		// if (type === "funnel-pre") {
+
+		// }
+
+		// if (type === "scd") {
+
+		// }
+
+		// if (type === "everything") {
+		// 	debugger;
+		// }
 
-		},
-		{
-			key: "video_id",
-			entries: 10_000_000,
-			attributes: {
-				isFlagged: [true, false, false, false, false],
-				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
-				uploader_id: chance.guid.bind(chance),
-				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
-				thumbs: weighNumRange(0, 35),
-				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
-			}
-
-		}],
-	hook: function (record, type, meta) {
 		return record;
 	}
 };
 
 
+function buildObjectProp() {
+	return function () {
+		return {
+			"foo key": choose(pickAWinner(["red", "orange", "yellow", "green", "blue", "indigo", "violet"])()),
+			"bar key": integer(1, 100),
+			"baz key": choose(pickAWinner([true, true, false])()),
+		};
+	};
+}
+
+
+function buildNumberArray() {
+	return function () {
+		const arr = [];
+		const times = integer(1, 10);
+		for (let i = 0; i < times; i++) {
+			arr.push(integer(1, 100));
+		}
+		return [arr];
+	};
+}
+
+function buildStringArray() {
+	return function () {
+		const arr = [];
+		const times = integer(1, 10);
+		for (let i = 0; i < times; i++) {
+			arr.push(makeName(integer(1, 3), " "));
+		}
+		return [arr];
+	};
+}
+
+
+function buildObjArrayProp(maxItems = 5) {
+
+	return function () {
+		const categories = ["Device Accessories", "eBooks", "Automotive", "Baby Products", "Beauty", "Books", "Camera & Photo", "Cell Phones & Accessories", "Collectible Coins", "Consumer Electronics", "Entertainment Collectibles", "Fine Art", "Grocery & Gourmet Food", "Health & Personal Care", "Home & Garden", "Independent Design", "Industrial & Scientific", "Accessories", "Major Appliances", "Music", "Musical Instruments", "Office Products", "Outdoors", "Personal Computers", "Pet Supplies", "Software", "Sports", "Sports Collectibles", "Tools & Home Improvement", "Toys & Games", "Video, DVD & Blu-ray", "Video Games", "Watches"];
+		const slugs = ['/sale/', '/featured/', '/home/', '/search/', '/wishlist/', '/'];
+		const assetExtension = ['.png', '.jpg', '.jpeg', '.heic', '.mp4', '.mov', '.avi'];
+		const data = [];
+		const numOfItems = integer(1, maxItems);
+
+		for (var i = 0; i < numOfItems; i++) {
+			const category = chance.pickone(categories);
+			const slug = chance.pickone(slugs);
+			const asset = chance.pickone(assetExtension);
+			const price = integer(1, 300);
+			const quantity = integer(1, 5);
+
+			const item = {
+				sku: integer(11111, 99999),
+				amount: price,
+				quantity: quantity,
+				total: price * quantity,
+				featured: chance.pickone([true, false]),
+				category: category,
+				urlSlug: slug + category,
+				asset: `${category}-${integer(1, 20)}${asset}`
+			};
+
+			data.push(item);
+		}
+
+		return () => [data];
+	};
+};
+
 
 module.exports = config;

package/dungeons/gaming-experiments.js

@@ -0,0 +1,323 @@
+
+const SEED = "my-seed";
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+require("dotenv").config();
+const u = require("../components/utils.js");
+const v = require("ak-tools");
+const chance = u.initChance(SEED);
+const num_users = 10_000;
+const days = 108;
+
+/** @typedef  {import("../types.js").Dungeon} Config */
+
+/** @type {Config} */
+const config = {
+	token: "",
+	seed: "twerk",
+	numDays: days,
+	numEvents: num_users * 100,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: false,
+	format: "json",
+	alsoInferFunnels: false,
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: false,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: false,
+	percentUsersBornInDataset: 35,
+
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 1_500_000,
+	concurrency: 1,
+	writeToDisk: false,
+	funnels: [
+		{
+			"sequence": ["app install", "character creation", "join party"],
+			"isFirstFunnel": true,
+			"conversionRate": 0.8,
+			"timeToConvert": .25,
+		},
+		{
+			"sequence": ["start quest", "gameplay summary", "complete quest"],
+			conversionRate: 0.99,
+			timeToConvert: 1,
+			props: {
+				"quest type": u.pickAWinner([
+					"Rescue",
+					"Retrieve",
+					"Explore",
+					"Destroy",
+					"Investigate",
+				]),
+				"quest difficulty": u.pickAWinner([
+					"Easy",
+					"Medium",
+					"Hard",
+					"Legendary",
+				]),
+				"quest location": u.pickAWinner([
+					"Forest",
+					"Dungeon",
+					"Mountain",
+					"City",
+					"Desert",
+				])
+			}
+		},
+	],
+	events: [
+		{ event: "app install", weight: 10, isFirstEvent: true },
+		{
+			event: "character creation",
+			weight: 2,
+			properties: {
+				mode: u.pickAWinner(["fast", "slow", "template"]),
+			},
+		},
+		{
+			event: "join party",
+			weight: 8,
+			properties: {
+				"party size": u.weighNumRange(1, 6),
+				"party leader": u.pickAWinner([
+					"Bard",
+					"Cleric",
+					"Fighter",
+					"Rogue",
+					"Wizard",
+					"Paladin",
+					"Ranger",
+					"Sorcerer",
+					"Warlock",
+				]),
+			},
+		},
+		{
+			event: "start quest",
+			weight: 12,
+			properties: {},
+		},
+		{
+			event: "complete quest",
+			weight: 12,
+			properties: {},
+		},
+		{
+			"event": "gameplay summary",
+			"weight": 14,
+			"properties": {
+				"# enemies defeated": u.weighNumRange(0, 100),
+				"# respawns": u.weighNumRange(0, 10, 5),
+				"# attacks": u.weighNumRange(0, 100, 6, 12),
+				"# gold found": u.weighNumRange(0, 1000),
+			}
+		},
+		{
+			"event": "in-game purchase",
+			"weight": 2,
+			"properties": {
+				"purchase type": u.pickAWinner([
+					"Gold",
+					"Gems",
+					"Items: Weapons",
+					"Items: Armor",
+					"Items: Potions",
+					"Items: Scrolls",
+					"Boosts",
+					"Currency",
+				]),
+				"purchase amount": u.weighNumRange(1, 50, 1, 20),
+			}
+		},
+		{
+			event: "fight boss",
+			weight: 3,
+			properties: {
+				"boss type": u.pickAWinner([
+					"Dragon",
+					"Demon",
+					"Lich",
+					"Vampire",
+					"Beholder",
+				]),
+				"boss level": u.weighNumRange(10, 20),
+				"boss difficulty": u.pickAWinner([
+					"Hard",
+					"Legendary",
+					"Impossible",
+				]),
+				"fight duration (mins)": u.weighNumRange(1, 60),
+
+			},
+		},
+		{
+			event: "level up",
+			weight: 1,
+			properties: {
+				"new abilities": u.pickAWinner([
+					"Attack",
+					"Spell",
+					"Feat",
+					"Skill",
+				]),
+			},
+		},
+		// {
+		// 	event: "generic event (common)",
+		// 	weight: 5,
+		// 	properties: {
+		// 		generic_prop: u.pickAWinner(["foo", "bar", "baz", "qux"]),
+		// 	}
+		// },
+		{
+			event: "$experiment_started",
+			weight: 3,
+			properties: {
+				"Experiment name": u.pickAWinner([
+					"fast leveling",
+					"tension economy",
+					"free trial",
+				]),
+				"Variant": ["A", "B", "C", "Control"],
+			}
+		},
+	],
+	superProps: {
+		platform: u.pickAWinner([
+			"Mobile",
+			"Xbox",
+			"Playstation",
+			"Switch",
+			"Web"
+		]),
+		"game mode": u.pickAWinner([
+			"Single Player",
+			"Multiplayer",
+		], 1),
+		language: u.pickAWinner([
+			"English",
+			"Spanish",
+			"French",
+			"German",
+			"Japanese",
+			"Korean",
+			"Chinese",
+		], 0),
+
+	},
+	scdProps: {
+		// "subscription MRR" : {
+		// 	"frequency": "week",
+		// 	"type": "number",
+		// 	values: u.weighNumRange(0, 250, 1, 200),
+		// 	timing: "fixed",
+		// 	max: 250,			
+		// }
+	},
+	userProps: {
+
+		race: u.pickAWinner([
+			"Human",
+			"Elf",
+			"Dwarf",
+			"Halfling",
+			"Dragonborn",
+			"Gnome",
+			"Half-Elf",
+			"Half-Orc",
+			"Tiefling",
+		]),
+		class: u.pickAWinner([
+			"Barbarian",
+			"Bard",
+			"Cleric",
+			"Druid",
+			"Fighter",
+			"Monk",
+			"Paladin",
+			"Ranger",
+			"Rogue",
+			"Sorcerer",
+			"Warlock",
+			"Wizard",
+		]),
+		alignment: u.pickAWinner([
+			"Lawful Good",
+			"Neutral Good",
+			"Chaotic Good",
+			"Lawful Neutral",
+			"True Neutral",
+			"Chaotic Neutral",
+			"Lawful Evil",
+			"Neutral Evil",
+			"Chaotic Evil",
+		]),
+		level: u.weighNumRange(1, 20),
+		background: u.pickAWinner([
+			"Acolyte",
+			"Charlatan",
+			"Criminal",
+			"Entertainer",
+			"Folk Hero",
+			"Guild Artisan",
+			"Hermit",
+			"Noble",
+			"Outlander",
+			"Sage",
+			"Sailor",
+			"Soldier",
+			"Urchin",
+		]),
+		"subscription MRR": u.weighNumRange(0, 250, 1, 200)
+	},
+	hook: function (record, type, meta) {
+		// const NOW = dayjs();
+
+		if (type === "event") {
+			// const EVENT_TIME = dayjs(record.time);
+		}
+
+		if (type === "user") {
+
+		}
+
+		if (type === "funnel-post") {
+
+		}
+
+		if (type === "funnel-pre") {
+			const sequence = v.clone(record.sequence);
+			if (sequence.includes("$experiment_started")) {
+				//ensure there's only one experiment per user flow
+				const newSequence = sequence.filter((event) => event !== "$experiment_started");
+
+				newSequence.unshift("$experiment_started");
+
+
+				record.sequence = newSequence;
+			}
+
+		}
+
+		if (type === "scd") {
+
+		}
+
+		if (type === "everything") {
+			
+		}
+
+		return record;
+	}
+};
+
+module.exports = config;

package/dungeons/simple.js

@@ -118,8 +118,7 @@ const config = {
 			}
 		}
 	],
-	superProps: {
-		platform: ["web", "mobile", "web", "mobile", "web", "web", "kiosk", "smartTV"],
+	superProps: {		
 		currentTheme: weighChoices(["light", "dark", "custom", "light", "dark"]),
 	},
 	/*