make-mp-data 2.1.6 → 3.0.1

package/README.md

@@ -87,6 +87,37 @@ Here's a breakdown of the CLI options you can use with `make-mp-data`:
 - `--complex`: create a complex set models including groups, SCD, and lookup tables.
 - `--simple`: create a simple dataset including events, and users
 
+## 🎯 Hooks — Engineering Trends in Data
+
+Hooks let you engineer deliberate, discoverable patterns into your generated data — things like "premium users convert 2x better" or "there was a service outage during days 40-47." A hook is a single transform function on your dungeon config:
+
+```javascript
+const config = {
+  // ...events, funnels, etc.
+  hook: function(record, type, meta) {
+    // Modify events based on type
+    if (type === "event" && record.event === "purchase") {
+      record.amount = record.amount * 2; // boost purchase amounts
+    }
+    
+    // Use "everything" to correlate across a user's full event stream
+    if (type === "everything") {
+      const hasPurchase = record.some(e => e.event === "purchase");
+      for (const event of record) {
+        event.is_buyer = hasPurchase; // tag all events for this user
+      }
+      return record;
+    }
+    
+    return record;
+  }
+};
+```
+
+Hook types: `event`, `user`, `everything`, `funnel-pre`, `funnel-post`, `scd-pre`, `ad-spend`, `group`, `mirror`, `lookup`.
+
+See `dungeons/harness/` for comprehensive examples with 8 hooks each across gaming, fintech, education, food delivery, media, social, and SaaS verticals.
+
 ## 📄 Examples
 
 Check out the examples directory for sample data models:

package/dungeons/adspend.js

@@ -14,11 +14,11 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
 import { uid, comma } from 'ak-tools';
-import { pickAWinner, weighNumRange, date, integer } from '../lib/utils/utils.js';
+import { pickAWinner, weighNumRange, date, integer } from "../lib/utils/utils.js";
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: "foo bar",
 	numDays: 365, //how many days worth of data
 	numEvents: 10000000, //how many events

package/dungeons/ai-chat-analytics-ed.js

@@ -15,7 +15,7 @@ const days = 100;
 
 /** @type {Dungeon} */
 const dungeon = {
-    token: "",
+    token: process.env.MASTER_PROJECT_TOKEN || "",
     seed: SEED,
     numDays: days, 
     numEvents: num_users * 100, 
@@ -34,8 +34,9 @@ const dungeon = {
     hasAdSpend: true,
     
     hasAvatar: true,
+    makeChart: false,
 
-    batchSize: 1_500_000,
+    batchSize: 2_500_000,
     concurrency: 50,
     writeToDisk: false,
     

package/dungeons/anon.js

@@ -14,13 +14,13 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
 import { uid, comma } from 'ak-tools';
-import { pickAWinner, weighNumRange, date, integer } from '../lib/utils/utils.js';
+import { pickAWinner, weighNumRange, date, integer } from "../lib/utils/utils.js";
 
 
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: "foo bar",
 	numDays: 365, //how many days worth of data
 	numEvents: 100000, //how many events

package/dungeons/array-of-object-loopup.js

@@ -0,0 +1,181 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+import Chance from 'chance';
+let chance = new Chance();
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+dayjs.extend(utc);
+import { uid, comma } from 'ak-tools';
+import { pickAWinner, weighNumRange, date, integer, weighChoices } from "../lib/utils/utils.js";
+
+const videoCategories = ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"];
+const spiritAnimals = ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"];
+
+/** @type {import('../types.js').Dungeon} */
+const config = {
+	// token: "0a3e6aa01225ed03856dca545c2b5b3d",
+	seed: "test array of objects lookup",
+	name: "array-of-object-lookup",
+	numDays: 60, //how many days worth1 of data
+	numEvents: 100_000, //how many events
+	numUsers: 1_000, //how many users
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
+	hasAdSpend: false,
+	makeChart: false,
+	hasLocation: true,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: false,
+	isAnonymous: false,
+	alsoInferFunnels: true,
+	concurrency: 1,
+	batchSize: 250_000,
+	writeToDisk: true,
+	events: [
+		{
+			event: "checkout",
+			weight: 2,
+			properties: {
+				currency: pickAWinner(["USD", "CAD", "EUR", "BTC", "ETH", "JPY"], 0),
+				coupon: weighChoices(["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"]),
+				cart: makeProducts()
+			}
+		},
+		{
+			event: "add to cart",
+			weight: 4,
+			properties: {
+				item: makeProducts(1),
+			}
+		},
+		{
+			event: "view item",
+			weight: 8,
+			properties: {
+				item: makeProducts(1)
+			}
+		},
+		{
+			event: "save item",
+			weight: 5,
+			properties: {
+				item: makeProducts(1),
+			}
+		}		
+	],
+	funnels: [	],
+	superProps: {
+		theme: pickAWinner(["light", "dark", "custom", "light", "dark"]),
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		// title: chance.profession.bind(chance),
+		// luckyNumber: weighNumRange(1, 500, .3),
+		spiritAnimal: spiritAnimals
+	},
+	scdProps: {},
+	mirrorProps: {},
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]]
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [{
+		key: "product_id",
+		entries: 1000,
+		attributes: {
+			amount: weighNumRange(1, 1000, .3),
+			quantity: weighNumRange(1, 10, .3),
+			featured: flip,
+			category: ["electronics", "books", "clothing", "home", "garden", "toys", "sports", "automotive", "beauty", "health", "grocery", "jewelry", "shoes", "tools", "office supplies"],
+			descriptor: ["brand new", "open box", "refurbished", "used", "like new", "vintage", "antique", "collectible"]
+			
+		}
+
+	}],
+	hook: function (record, type, meta) {
+
+		const NOW = dayjs();
+
+
+		if (type === "event") {
+			
+		}
+
+		if (type === "everything") {
+			
+
+		}
+
+
+
+		return record;
+	}
+};
+
+function makeProducts(maxItems = 5) {
+	return function () {
+		const categories = ["electronics", "books", "clothing", "home", "garden", "toys", "sports", "automotive", "beauty", "health", "grocery", "jewelry", "shoes", "tools", "office supplies"];
+		const descriptors = ["brand new", "open box", "refurbished", "used", "like new", "vintage", "antique", "collectible"];
+		const suffix = ["item", "product", "good", "merchandise", "thing", "object", "widget", "gadget", "device", "apparatus", "contraption", "instrument", "tool", "implement", "utensil", "appliance", "machine", "equipment", "gear", "kit", "set", "package"];
+		const assetPreview = ['.png', '.jpg', '.jpeg', '.heic', '.mp4', '.mov', '.avi'];
+		const data = [];
+		const numOfItems = integer(1, maxItems);
+
+		for (var i = 0; i < numOfItems; i++) {
+			const category = chance.pickone(categories);
+			const descriptor = chance.pickone(descriptors);
+			const suffixWord = chance.pickone(suffix);
+			const slug = `${descriptor.replace(/\s+/g, '-').toLowerCase()}-${suffixWord.replace(/\s+/g, '-').toLowerCase()}`;
+			const asset = chance.pickone(assetPreview);
+
+			// const product_id = chance.guid();
+			const price = integer(1, 100);
+			const quantity = integer(1, 5);
+			const product_id = integer(1, 1_000);
+
+			const item = {
+				product_id: product_id,
+				product_url: `https://example.com/assets/${product_id}`,
+				// sku: integer(11111, 99999),
+				// amount: price,
+				// quantity: quantity,
+				// total_value: price * quantity,
+				// featured: chance.pickone([true, false, false]),
+				// category: category,
+				// descriptor: descriptor,
+				// slug: slug,
+				
+				// assetType: asset
+
+			};
+
+			data.push(item);
+		}
+
+		return () => [data];
+	};
+};
+
+
+function flip(likelihood = 50) {
+	return chance.bool({ likelihood });
+}
+
+
+export default config;

package/dungeons/benchmark-heavy.js

@@ -0,0 +1,241 @@
+/**
+ * Benchmark Dungeon Configuration
+ * Generates one million events across 10,000 users
+ * ensure we have multiple property types and distributions
+ */
+
+
+
+
+import Chance from 'chance';
+let chance = new Chance();
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+dayjs.extend(utc);
+import { uid, comma } from 'ak-tools';
+import { pickAWinner, weighNumRange, date, integer, weighChoices } from "../lib/utils/utils.js";
+import { createTextGenerator } from '../lib/generators/text.js';
+
+// list of strings use case
+function buildListOfFeatures() {
+	const fullFeatureList = ["beta flags", "alpha flags", "custom themes", "API access", "webhooks", "data export", "data import", "external integrations", "private cloud", "multi factor auth", "audit logs", "custom domain", "personalized landing"];
+	const numFeatures = integer(3, 7);
+	const selectedFeatures = chance.pickset(fullFeatureList, numFeatures);
+	return [selectedFeatures];
+}
+
+// object use case
+function buildLocalConfig() {
+	const allPossibleConfigs = {
+		"theme": pickAWinner(["light", "dark", "custom"]),
+		"notifications": pickAWinner(["all", "mentions", "none"]),
+		"itemsPerPage": pickAWinner([10, 25, 50, 100]),
+		"language": pickAWinner(["en", "es", "fr", "de", "zh", "jp", "ru"]),
+		"timezone": pickAWinner(["UTC", "PST", "EST", "CST", "MST", "GMT", "CET", "IST", "JST", "AEST"]),
+		"privacyLevel": pickAWinner(["public", "friends only", "private"]),
+		"dataSharing": [true, false],
+		"autoSave": [true, false],
+		"fontSize": ["small", "medium", "large"],
+		"layout": ["grid", "list", "compact"]
+	};
+	const config = {};
+
+	for (const [key, values] of Object.entries(allPossibleConfigs)) {
+		let value;
+		if (typeof values === 'function') {
+			value = values();
+		} else if (Array.isArray(values)) {
+			value = values;
+		} else {
+			value = values;
+		}
+		config[key] = chance.pickone(value);
+	}
+	return config;
+}
+
+
+// array of object use case
+function buildCart() {
+	const allPossibleItems = [
+		{ itemId: uid(), name: "widget foo", price: 9.99, category: "Gadgets", isPremium: false, stock: 150 },
+		{ itemId: uid(), name: "poker bar", price: 14.99, category: "Gizmos", isPremium: true, stock: 100 },
+		{ itemId: uid(), name: "limited baz", price: 4.99, category: "Toys", isPremium: false, stock: 200 },
+		{ itemId: uid(), name: "foolish qux", price: 19.99, category: "Tools", isPremium: false, stock: 50 },
+		{ itemId: uid(), name: "topical mux", price: 29.99, category: "Components", isPremium: true, stock: 75 },
+		{ itemId: uid(), name: "random garply", price: 24.99, category: "Gadgets", isPremium: false, stock: 30 },
+		{ itemId: uid(), name: "helpful waldo", price: 39.99, category: "Gizmos", isPremium: true, stock: 20 },
+		{ itemId: uid(), name: "hillarious defcon", price: 11.99, category: "Toys", isPremium: false, stock: 120 },
+		{ itemId: uid(), name: "final radiator", price: 7.99, category: "Tools", isPremium: false, stock: 80 },
+		{ itemId: uid(), name: "leaning wholesome", price: 49.99, category: "Components", isPremium: true, stock: 10 },
+		{ itemId: uid(), name: "whose friends", price: 17.99, category: "Gadgets", isPremium: false, stock: 60 },
+	];
+	const numItems = integer(1, 20);
+	const selectedItems = chance.pickset(allPossibleItems, numItems);
+	// add quantity to each item
+	for (const item of selectedItems) {
+		item.quantity = integer(1, 8);
+		item.totalPrice = item.price * item.quantity;
+	}
+	return [...selectedItems];
+
+}
+
+
+
+const commentGenerator = createTextGenerator({
+	authenticityLevel: 0.8,
+	enableDeduplication: true,
+	includeMetadata: false,
+	style: "comments",
+	formality: "casual",
+	mixedSentiment: true,
+	keywords: {
+		"brands": ["Nike", "Adidas", "Puma", "Reebok", "Under Armour", "New Balance", "Asics", "Skechers", "Converse", "Vans"],
+		"categories": ["shoes", "sneakers", "running shoes", "basketball shoes", "casual shoes", "formal shoes", "boots", "sandals", "heels", "flats"],
+		"credibility": ["verified purchase", "top reviewer", "frequent buyer", "long-time customer", "new customer"],
+		"features": ["comfort", "durability", "style", "fit", "value for money", "design", "color options", "material quality", "breathability", "support"],
+		"issues": ["size runs small", "poor arch support", "slippery sole", "uncomfortable", "not true to color", "weak laces", "easily scuffed", "bad odor", "heavy weight", "limited stock"],
+	},
+	typos: false,
+	min: 20,
+	max: 255,
+
+});
+
+
+
+/** @type {import('../types.js').Dungeon} */
+const config = {
+	// token: process.env.MASTER_PROJECT_TOKEN || "",
+	name: "300k-Events-Heavy",
+	format: 'json', //csv or json
+	seed: "one million events",
+	numDays: 92, //how many days worth of data
+	numEvents: 300_000, //how many events
+	numUsers: 10_000, //how many users
+	strictEventCount: true,
+
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
+	hasAdSpend: false,
+	makeChart: false,
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	alsoInferFunnels: true,
+	// concurrency automatically set to 1 when strictEventCount is enabled
+	writeToDisk: true,
+	batchSize: 2_500_000,
+
+	events: [
+		{
+			event: "Page View",
+			weight: 50,
+
+		},
+		{
+			event: "Product View",
+			weight: 30,
+		},
+		{
+			event: "Add to Cart",
+			weight: 10,
+		},
+		{
+			event: "Checkout Started",
+			weight: 5,
+		},
+		{
+			event: "Purchase",
+			weight: 1,
+		},
+		{
+			event: "Product Review",
+			weight: 3,
+		},
+		{
+			event: "Search",
+			weight: 17,
+		},
+		{
+			event: "Browse",
+			weight: 25,
+
+		},
+		{
+			event: "Save for Later",
+			weight: 4,
+		},
+		{
+			event: "Remove from Cart",
+			weight: 2,
+		},
+		{
+			event: "Empty Cart",
+			weight: 4,
+		}
+	],
+	funnels: [
+		{
+			"name": "Purchase Funnel",
+			"sequence": [
+				"Page View",
+				"Product View",
+				"Add to Cart",
+				"Checkout Started",
+				"Purchase"
+			],
+			requireRepeats: true,
+			"conversionRate": 30,
+			"order": "fixed",
+			"weight": 1,
+			"isFirstFunnel": false,
+			"timeToConvert": 5,
+			"experiment": true,
+		}
+	],
+	superProps: {
+		"theme (string)": pickAWinner(["light", "dark", "custom", "light", "dark"]),
+		"latency (number)": weighNumRange(20, 600, 0.7),
+		"beta user (boolean)": [true, false, false, false],
+		"birthdate (date)": date(30000, true, 'YYYY-MM-DD'),
+		"features (list)": buildListOfFeatures,
+		"config (object)": buildLocalConfig,
+		"cart (array of objects)": buildCart,
+		"comment (text)": commentGenerator,
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		
+
+	},
+	scdProps: {},
+	mirrorProps: {},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		// if (type === "everything") {
+		// 	debugger;
+		// }
+		return record;
+	}
+};
+
+
+
+export default config;

package/dungeons/benchmark-light.js

@@ -0,0 +1,141 @@
+/**
+ * Benchmark Dungeon Configuration
+ * Generates one million events across 10,000 users
+ * ensure we have multiple property types and distributions
+ */
+
+
+
+
+import Chance from 'chance';
+let chance = new Chance();
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+dayjs.extend(utc);
+import { uid, comma } from 'ak-tools';
+import { pickAWinner, weighNumRange, date, integer, weighChoices } from "../lib/utils/utils.js";
+import { createTextGenerator } from '../lib/generators/text.js';
+
+/** @type {import('../types.js').Dungeon} */
+const config = {
+	// token: process.env.MASTER_PROJECT_TOKEN || "",
+	name: "5M-Events-Light",
+	format: 'json', //csv or json
+	seed: "one million events",
+	numDays: 92, //how many days worth of data
+	numEvents: 5_000_000, //how many events
+	numUsers: 10_000, //how many users
+	strictEventCount: true,
+	
+	region: "US",
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	hasAdSpend: false,
+	makeChart: false,
+	hasLocation: false,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: false,
+	hasBrowser: false,
+	hasCampaigns: false,
+	isAnonymous: false,
+	alsoInferFunnels: true,
+	// concurrency automatically set to 1 when strictEventCount is enabled
+	writeToDisk: true,
+	batchSize: 12_500_000,
+
+	events: [
+		{
+			event: "Page View",
+			weight: 50,
+
+		},
+		{
+			event: "Product View",
+			weight: 30,
+		},
+		{
+			event: "Add to Cart",
+			weight: 10,
+		},
+		{
+			event: "Checkout Started",
+			weight: 5,
+		},
+		{
+			event: "Purchase",
+			weight: 1,
+		},
+		{
+			event: "Product Review",
+			weight: 3,
+		},
+		{
+			event: "Search",
+			weight: 17,
+		},
+		{
+			event: "Browse",
+			weight: 25,
+
+		},
+		{
+			event: "Save for Later",
+			weight: 4,
+		},
+		{
+			event: "Remove from Cart",
+			weight: 2,
+		},
+		{
+			event: "Empty Cart",
+			weight: 4,
+		}
+	],
+	funnels: [
+		{
+			"name": "Purchase Funnel",
+			"sequence": [
+				"Page View",
+				"Product View",
+				"Add to Cart",
+				"Checkout Started",
+				"Purchase"
+			],
+			requireRepeats: true,
+			"conversionRate": 30,
+			"order": "fixed",
+			"weight": 1,
+			"isFirstFunnel": false,
+			"timeToConvert": 5,
+			"experiment": true,
+		}
+	],
+	superProps: {
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {},
+	scdProps: {},
+	mirrorProps: {},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		// if (type === "everything") {
+		// 	debugger;
+		// }
+		return record;
+	}
+};
+
+
+
+export default config;