make-mp-data 3.0.6 → 3.1.0

package/dungeons/complex.js

@@ -0,0 +1,428 @@
+/**
+ * This is the default configuration file for the data generator in COMPLEX mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+import Chance from 'chance';
+const chance = new Chance();
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+dayjs.extend(utc);
+import { weighNumRange, date, integer } from "../lib/utils/utils.js";
+import * as u from 'ak-tools';
+
+/** @type {import('../types.js').Dungeon} */
+const config = {
+	token: "",
+	seed: "quite complexus",
+	numDays: 30, //how many days worth of data
+	numEvents: 100_000, //how many events
+	numUsers: 1000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
+
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+
+	hasAvatar: true,
+
+	batchSize: 500_000,
+	concurrency: 1,
+
+	funnels: [],
+	events: [
+		{
+			"event": "checkout",
+			"weight": 2,
+			"properties": {
+				amount: weighNumRange(5, 500, .25),
+				currency: ["USD", "USD", "USD", "CAD", "EUR", "EUR", "BTC", "BTC", "ETH", "JPY"],
+				cart: makeProducts(12),
+			}
+		},
+		{
+			"event": "add to cart",
+			"weight": 4,
+			"properties": {
+				amount: weighNumRange(5, 500, .25),
+				qty: integer(1, 5),
+				product_id: weighNumRange(1, 1000, 1.4)
+			}
+		},
+		{
+			"event": "page view",
+			"weight": 10,
+			"properties": {
+				page: ["/", "/", "/", "/learn-more", "/pricing", "/contact", "/about", "/careers", "/sign-up", "/login", "/app", "/app", "/app", "/app"],
+				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
+			}
+		},
+		{
+			"event": "watch video",
+			"weight": 8,
+			"properties": {
+				category: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
+				hashTags: makeHashTags,
+				watchTimeSec: weighNumRange(10, 600, .25,),
+				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
+				format: ["mp4", "avi", "mov", "mpg"],
+				video_id: weighNumRange(1, 50000, 1.4),
+
+			}
+		},
+		{
+			"event": "comment",
+			"weight": 2,
+			"properties": {
+				length: weighNumRange(1, 500, .25),
+				video_id: weighNumRange(1, 50000, 1.4),
+				has_replies: [true, false, false, false, false],
+				has_photo: [true, false, false, false, false],
+
+			}
+		},
+		{
+			"event": "save video",
+			"weight": 4,
+			"properties": {
+				video_id: weighNumRange(1, 50000, 1.4),
+				ui_control: ["toolbar", "menu", "keyboard"]
+
+
+			}
+		},
+		{
+			"event": "view item",
+			"weight": 8,
+			"properties": {
+				product_id: weighNumRange(1, 24, 3),
+				colors: ["light", "dark", "custom", "dark"]
+			}
+		},
+		{
+			"event": "save item",
+			"weight": 5,
+			"properties": {
+				product_id: weighNumRange(1, 1000, 12),
+				colors: ["light", "dark", "custom", "dark"]
+			}
+		},
+		{
+			"event": "support ticket",
+			"weight": 2,
+			"properties": {
+				product_id: weighNumRange(1, 1000, .6),
+				description: chance.sentence.bind(chance),
+				severity: ["low", "medium", "high"],
+				ticket_id: chance.guid.bind(chance)
+			}
+		},
+		{
+			"event": "sign up",
+			"isFirstEvent": true,
+			"weight": 0,
+			"properties": {
+				plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
+				dateOfRenewal: date(100, false),
+				codewords: u.makeName,
+			}
+		}
+	],
+	superProps: {
+		linked_device: deviceAttributes()
+		// emotions: generateEmoji(),
+
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		experiment: designExperiment(),
+		spiritAnimal: ["unicorn", "dragon", "phoenix", "sasquatch", "yeti", "kraken", "jackalope", "thunderbird", "mothman", "nessie", "chupacabra", "jersey devil", "bigfoot", "weindgo", "bunyip", "mokele-mbembe", "tatzelwurm", "megalodon"],
+		timezone: chance.timezone.bind(chance), // ["America/New_York", "America/Los_Angeles", "America/Chicago", "America/Denver", "America/Phoenix", "America/Anchorage", "Pacific/Honolulu"]
+		ip: chance.ip.bind(chance),
+		lastCart: makeProducts(5),
+
+	},
+
+	/** each generates it's own table */
+	scdProps: {
+		role: {
+			type: "user",
+			frequency: "week",
+			values: ["admin", "collaborator", "user", "view only", "no access"],
+			timing: 'fuzzy',
+			max: 10
+		},
+		NPS: {
+			type: "user",
+			frequency: "day",
+			values: weighNumRange(1, 10, 2, 150),
+			timing: 'fuzzy',
+			max: 10
+		},
+		MRR: {
+			type: "company_id",
+			frequency: "month",
+			values: weighNumRange(0, 10000, .15),
+			timing: 'fixed',
+			max: 10
+		},
+		AccountHealthScore: {
+			type: "company_id",
+			frequency: "week",
+			values: weighNumRange(1, 10, .15),
+			timing: 'fixed',
+			max: 40
+		},
+		plan: {
+			type: "company_id",
+			frequency: "month",
+			values: ["free", "basic", "premium", "enterprise"],
+			timing: 'fixed',
+			max: 10
+		}
+	},
+
+	mirrorProps: {
+		isBot: { events: "*", values: [false, false, false, false, true] },
+		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
+		watchTimeSec: {
+			events: ["watch video"],
+			values: weighNumRange(50, 1200, 2)
+		}
+	},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [
+		['company_id', 500, []],
+		['room_id', 10000, ["save video", "comment", "watch video"]],
+
+	],
+	groupProps: {
+		company_id: {
+			name: () => { return chance.company(); },
+			email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
+			"# of employees": weighNumRange(3, 10000),
+			"industry": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
+			"segment": ["enterprise", "SMB", "mid-market"],
+			"products": [["core"], ["core"], ["core", "add-ons"], ["core", "pro-serve"], ["core", "add-ons", "pro-serve"], ["core", "BAA", "enterprise"], ["free"], ["free"], ["free", "addons"]],
+		},
+		room_id: {
+			name: () => { return `#${chance.word({ length: integer(4, 24), capitalize: true })}`; },
+			email: ["public", "private"],
+			"room provider": ["partner", "core", "core", "core"],
+			"room capacity": weighNumRange(3, 1000000),
+			"isPublic": [true, false, false, false, false],
+			"country": chance.country.bind(chance),
+			"isVerified": [true, true, false, false, false],
+		}
+	},
+	groupEvents: [{
+		attribute_to_user: false,
+		event: "card charged",
+		weight: 1,
+		frequency: 30,
+		group_key: "company_id",
+		group_size: 500,
+		properties: {
+			amount: weighNumRange(5, 500, .25),
+			currency: ["USD", "USD", "USD", "CAD", "EUR", "EUR", "BTC", "BTC", "ETH", "JPY"],
+			plan: ["basic", "premium", "enterprise"],
+			"payment method": []
+		}
+	}],
+
+	lookupTables: [
+		{
+			key: "product_id",
+			entries: 1000,
+			attributes: {
+				category: [
+					"Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden & Tools", "Pet Supplies", "Food & Grocery", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art"
+				],
+				"demand": ["high", "medium", "medium", "low"],
+				"supply": ["high", "medium", "medium", "low"],
+				"manufacturer": chance.company.bind(chance),
+				"price": weighNumRange(5, 500, .25),
+				"rating": weighNumRange(1, 5),
+				"reviews": weighNumRange(0, 35)
+			}
+
+		},
+		{
+			key: "video_id",
+			entries: 50000,
+			attributes: {
+				isFlagged: [true, false, false, false, false],
+				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
+				uploader_id: chance.guid.bind(chance),
+				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
+				thumbs: weighNumRange(0, 35),
+				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
+			}
+
+		}
+	],
+
+	hook: function (record, type, meta) {
+		// event hook: weekend watch time boost — videos watched on weekends get 1.5x duration
+		if (type === "event") {
+			if (record.event === "watch video" && record.time) {
+				const day = dayjs(record.time).day();
+				if (day === 0 || day === 6) {
+					record.watchTimeSec = Math.round((record.watchTimeSec || 60) * 1.5);
+					record.is_weekend = true;
+				}
+			}
+			// support tickets on high-severity get escalated flag
+			if (record.event === "support ticket" && record.severity === "high") {
+				record.escalated = true;
+			}
+		}
+
+		// everything hook: simulate cart abandonment — users who "add to cart" but never "checkout" lose their last add-to-cart
+		if (type === "everything") {
+			const hasCheckout = record.some(e => e.event === "checkout");
+			const hasAddToCart = record.some(e => e.event === "add to cart");
+			if (hasAddToCart && !hasCheckout) {
+				// mark all their add-to-cart events as abandoned
+				for (const e of record) {
+					if (e.event === "add to cart") {
+						e.abandoned = true;
+					}
+				}
+			}
+			return record;
+		}
+
+		return record;
+	}
+};
+
+
+
+function makeHashTags() {
+	const possibleHashtags = [];
+	for (let i = 0; i < 20; i++) {
+		possibleHashtags.push('#' + u.makeName(2, ''));
+	}
+
+	const numHashtags = integer(integer(1, 5), integer(5, 10));
+	const hashtags = [];
+	for (let i = 0; i < numHashtags; i++) {
+		hashtags.push(chance.pickone(possibleHashtags));
+	}
+	return [hashtags];
+};
+
+function makeProducts(maxItems = 10) {
+
+	return function () {
+		const categories = ["Device Accessories", "eBooks", "Automotive", "Baby Products", "Beauty", "Books", "Camera & Photo", "Cell Phones & Accessories", "Collectible Coins", "Consumer Electronics", "Entertainment Collectibles", "Fine Art", "Grocery & Gourmet Food", "Health & Personal Care", "Home & Garden", "Independent Design", "Industrial & Scientific", "Accessories", "Major Appliances", "Music", "Musical Instruments", "Office Products", "Outdoors", "Personal Computers", "Pet Supplies", "Software", "Sports", "Sports Collectibles", "Tools & Home Improvement", "Toys & Games", "Video, DVD & Blu-ray", "Video Games", "Watches"];
+		const slugs = ['/sale/', '/featured/', '/home/', '/search/', '/wishlist/', '/'];
+		const assetExtension = ['.png', '.jpg', '.jpeg', '.heic', '.mp4', '.mov', '.avi'];
+		const data = [];
+		const numOfItems = integer(1, 12);
+
+		for (var i = 0; i < numOfItems; i++) {
+			const category = chance.pickone(categories);
+			const slug = chance.pickone(slugs);
+			const asset = chance.pickone(assetExtension);
+			const product_id = chance.guid();
+			const price = integer(1, 300);
+			const quantity = integer(1, 5);
+
+			const item = {
+				product_id: product_id,
+				sku: integer(11111, 99999),
+				amount: price,
+				quantity: quantity,
+				value: price * quantity,
+				featured: chance.pickone([true, false]),
+				category: category,
+				urlSlug: slug + category,
+				asset: `${category}-${integer(1, 20)}${asset}`
+			};
+
+			data.push(item);
+		}
+
+		return () => [data];
+	};
+};
+
+
+function designExperiment() {
+	return function () {
+		const variants = ["A", "B", "C", "Control"];
+		const variant = chance.pickone(variants);
+		const experiments = ["no password", "social sign in", "new tutorial", "new search"];
+		const experiment = chance.pickone(experiments);
+		const multi_variates = ["A/B", "A/B/C", "A/B/C/D", "Control"];
+		const multi_variate = chance.pickone(multi_variates);
+		const impression_id = chance.guid();
+
+
+
+		const chosen = {
+			variant,
+			experiment,
+			multi_variate,
+			impression_id
+		};
+
+		return [chosen];
+	};
+}
+
+function deviceAttributes(isMobile = false) {
+	return function () {
+		let devices = ["desktop", "laptop", "desktop", "laptop", "desktop", "laptop", "other"];
+		if (isMobile) devices = [...devices, "mobile", "mobile", "mobile", "tablet"];
+		const device = chance.pickone(devices);
+		let oses = ["Windows", "macOS", "Windows", "macOS", "macOS", "Linux", "Windows", "macOS", "Windows", "macOS", "macOS", "TempleOS"];
+		if (isMobile) oses = [...oses, "iOS", "Android", "iOS", "Android"];
+		const os = chance.pickone(oses);
+		const browser = chance.pickone(["Chrome", "Firefox", "Safari", "Edge", "Opera", "IE", "Brave", "Vivaldi"]);
+		const version = chance.integer({ min: 1, max: 15 });
+		const resolution = chance.pickone(["1920x1080", "1280x720", "1024x768", "800x600", "640x480"]);
+		const language = chance.pickone(["en-US", "en-US", "en-US", "en-GB", "es", "es", "fr", "de", "it", "ja", "zh", "ru"]);
+
+		const chosen = {
+			platform: device,
+			os,
+			browser,
+			version,
+			resolution,
+			language
+		};
+
+		return chosen;
+
+	};
+}
+
+
+
+
+
+
+export default config;

package/entry.js

@@ -59,7 +59,7 @@ import getCliParams from './lib/cli/cli.js';
         process.exit(0);
     } catch (error) {
         console.error(`\n❌ Job failed: ${error.message}`);
-        if (process.env.NODE_ENV === 'development') {
+        if (process.env.NODE_ENV !== 'production') {
             console.error(error.stack);
         }
         process.exit(1);

package/index.js

@@ -34,6 +34,8 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import { timer } from 'ak-tools';
 import { existsSync } from 'fs';
+import path from 'path';
+import { pathToFileURL } from 'url';
 import { dataLogger as logger } from './lib/utils/logger.js';
 
 // Initialize dayjs and time constants
@@ -123,8 +125,11 @@ async function main(config) {
 				console.log(`\n🔍 Loading dungeon config from: ${firstArg}`);
 			}
 			try {
-				const dungeonConfig = await import(firstArg);
-				config = dungeonConfig.default || dungeonConfig;
+				const absolutePath = path.resolve(firstArg);
+				const dungeonConfig = await import(pathToFileURL(absolutePath).href);
+				const loaded = dungeonConfig.default || dungeonConfig;
+				// Merge: dungeon config provides defaults; CLI flags override
+				config = { ...loaded, ...config };
 			} catch (error) {
 				console.error(`\n❌ Error loading dungeon config from ${firstArg}: ${error.message}`);
 				throw error;

package/lib/cli/cli.js

@@ -93,21 +93,18 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option("region", {
 			demandOption: false,
-			default: 'US',
 			alias: 'r',
-			describe: 'either US or EU or IN',
+			describe: 'either US or EU or IN (defaults to dungeon config or US)',
 			type: 'string'
 		})
 		.option('concurrency', {
 			alias: 'conn',
-			default: 10,
 			demandOption: false,
-			describe: 'concurrency level for data generation',
+			describe: 'concurrency level for data generation (defaults to dungeon config)',
 			type: 'number'
 		})
 		.options("complex", {
 			demandOption: false,
-			default: false,
 			describe: 'use complex data model (model all entities)',
 			alias: 'c',
 			type: 'boolean',
@@ -115,7 +112,6 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.options("simple", {
 			demandOption: false,
-			default: false,
 			describe: 'use simple data model (basic events and users)',
 			alias: 'simp',
 			type: 'boolean',
@@ -123,7 +119,6 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.options("sanity", {
 			demandOption: false,
-			default: false,
 			describe: 'run sanity checks on the generated data',
 			alias: 'san',
 			type: 'boolean',
@@ -131,15 +126,13 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option("writeToDisk", {
 			demandOption: false,
-			default: true,
-			describe: 'write data to disk',
+			describe: 'write data to disk (CLI defaults to true)',
 			alias: 'w',
 			type: 'boolean',
 			coerce: boolCoerce
 		})
 		.option("hasSessionIds", {
 			demandOption: false,
-			default: false,
 			describe: 'create session ids in the data',
 			alias: 'sid',
 			type: 'boolean',
@@ -147,7 +140,6 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 		})
 		.option("hasAnonIds", {
 			demandOption: false,
-			default: false,
 			describe: 'create anonymous ids in the data',
 			alias: 'aid',
 			type: 'boolean',

package/lib/core/config-validator.js

@@ -341,18 +341,15 @@ function transformSCDPropsWithoutCredentials(config) {
 
 	// Missing credentials - handle based on job type
 	if (!isUIJob) {
-		// For programmatic/CLI usage, throw an error if trying to send SCDs to Mixpanel without credentials
-		if (token) {
-			throw new Error(
-				'Configuration error: SCD properties are configured but service credentials are missing.\n' +
-				'To import SCD data to Mixpanel, you must provide:\n' +
-				'  - serviceAccount: Your Mixpanel service account username\n' +
-				'  - serviceSecret: Your Mixpanel service account secret\n' +
-				'  - projectId: Your Mixpanel project ID\n' +
-				'Without these credentials, SCD data cannot be imported to Mixpanel.'
+		// For programmatic/CLI usage: warn but continue. SCD files still generate;
+		// mixpanel-sender already gates SCD import on credential presence.
+		if (token && config.verbose !== false) {
+			console.warn(
+				'⚠️  SCD properties configured but service credentials missing. ' +
+				'SCD files will be generated but NOT imported to Mixpanel. ' +
+				'Provide serviceAccount + serviceSecret + projectId to enable SCD import.'
 			);
 		}
-		// If not sending to Mixpanel (no token), allow generation for testing
 		return;
 	}